diff --git a/.env.example b/.env.example index 119384b5..a4b640ac 100644 --- a/.env.example +++ b/.env.example @@ -11,7 +11,7 @@ SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id @@ -23,3 +23,13 @@ WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 +MAX_ACTIVE_JOBS=500 +JOB_LEASE_SECONDS=30 +JOB_BACKOFF_BASE=1s +JOB_BACKOFF_MAX=60s +WORKER_DECODE_BUDGET_BYTES=402653184 +WORKER_MEMORY=768M +WORKER_CPUS=2.0 +WORKER_GOMEMLIMIT=600MiB +PENDING_UPLOAD_GRACE=15m +FAULT_INJECTION=1 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..b2b5733f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +# syntax=docker/dockerfile:1 + +FROM golang:1.23.4-alpine3.21 AS build +WORKDIR /src + +COPY src/go.mod src/go.sum ./ +RUN go mod download + +COPY src/ ./ +ENV CGO_ENABLED=0 +RUN go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server + +FROM alpine:3.21.2 AS runtime +RUN addgroup -S app && adduser -S -G app app +WORKDIR /app +COPY --from=build /out/server /app/server +ARG APP_BUILD=dev +ENV APP_BUILD=$APP_BUILD +LABEL org.opencontainers.image.revision=$APP_BUILD +USER app +EXPOSE 8080 +ENV APP_HOST=0.0.0.0 APP_PORT=8080 +HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \ + CMD wget -q -O /dev/null http://127.0.0.1:8080/health/ready || exit 1 +ENTRYPOINT ["/app/server"] diff --git a/Makefile b/Makefile index 5d86d171..90f70fca 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,42 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: secrets up down ps unit worker-stop worker-start ops track1 + +secrets: + $(PYTHON) scripts/init_secrets.py + +up: secrets + docker compose --env-file .env up -d --build --wait + +down: + docker compose stop + +ps: + docker compose ps -a + +unit: + docker run --rm -v "$(CURDIR)/src:/src" -w /src golang:1.23.4-alpine3.21 go test ./... + +worker-stop: + docker compose stop worker + +worker-start: + docker compose start worker + +ops: + $(PYTHON) scripts/lab5ops.py worker-down --output evidence/worker-down.json + $(PYTHON) scripts/lab5ops.py api-crash --output evidence/api-crash.json + $(PYTHON) scripts/lab5ops.py effect-crash --output evidence/effect-crash.json + $(PYTHON) scripts/lab5ops.py ack-crash --output evidence/ack-crash.json + $(PYTHON) scripts/lab5ops.py retry --output evidence/retry.json + $(PYTHON) scripts/lab5ops.py replay --output evidence/replay.json + $(PYTHON) scripts/lab5ops.py invalid --output evidence/invalid.json + $(PYTHON) scripts/lab5ops.py hang --output evidence/hang.json + $(PYTHON) scripts/lab5ops.py dlq --output evidence/dlq.json + $(PYTHON) scripts/lab5ops.py limits --output evidence/limits.json + +track1: + $(PYTHON) scripts/track1/run_track1.py matrix --output evidence/track1/matrix.json + $(PYTHON) scripts/track1/run_track1.py backpressure --output evidence/track1/backpressure.json diff --git a/REPORT.md b/REPORT.md index e98f75e9..0014e90d 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,193 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Кораблев Петр, БВТ2304 +- Целевая оценка: **5** (уровень 3 + уровень 4 + **трек 1 «Очередь под перегрузкой»**) +- Commit/PR: ветка `solution/BVT2304-korablev` +- Использованные источники, библиотеки и помощь ИИ: Go 1.23.4, pgx/v5 5.7.2, go-redis/v9 9.7.0, minio-go/v7 7.0.82 (из lab4); **новые**: `github.com/rabbitmq/amqp091-go` 1.10.0 (AMQP-клиент), `golang.org/x/image` 0.23.0 (`draw.CatmullRom` для ресайза), `github.com/gen2brain/webp` 0.5.2 (libwebp, скомпилированный в WebAssembly и исполняемый через wazero 1.8.1 — кодирование WebP без cgo). Образы: PostgreSQL 17.2-alpine3.21, Valkey 8.0.2-alpine3.21, HAProxy 3.0.5-alpine, SeaweedFS 3.80, **RabbitMQ 4.0.5-management-alpine**, golang:1.23.4-alpine3.21 → alpine:3.21.2, для служебных проверок python:3.12.8-alpine3.21. Документация: RabbitMQ Reliability / Confirms / Quorum queues, EXIF 2.32 (тег Orientation 0x0112). Помощь ИИ использовалась при написании отчёта. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Prerequisites: Docker 29.x + Compose v2 (Docker Desktop/WSL2), Python 3.9+, Git. Go и make локально не нужны (сборка в Docker; без make команды ниже выполняются через `python`). На Windows тесты запускать с `PYTHONUTF8=1`. + +```sh +python3 scripts/init_secrets.py # = make secrets: secrets/* + .env из .env.example +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait # = make up; ждёт healthy и завершения migrate/s3-init +make check && make test # smoke через LB http://localhost:8080 +make unit # Go unit-тесты imaging (размеры, EXIF, alpha, лимиты) +make test-persistence ACTION_SCRIPT=scripts/restart.sh # Windows: scripts/restart.cmd +make test-failover ACTION_SCRIPT=scripts/stop-one.sh # Windows: scripts/stop-one.cmd +docker compose up -d --no-recreate --wait api # вернуть остановленную реплику +make ops # сценарии отказов уровней 3/4 -> evidence/*.json +make track1 # трек 1 -> evidence/track1/*.json (~15 мин) +``` + +- `init_secrets.py` дополнительно к lab4 создаёт `secrets/rabbitmq_auth.conf` (пользователь `messenger` со случайным паролем; монтируется как Compose secret в `/etc/rabbitmq/conf.d/20-auth.conf`, гостевой пользователь не создаётся) и `secrets/broker_url` (AMQP URL для API/worker через `BROKER_URL_FILE`). Всё в `secrets/` (в `.gitignore`/`.dockerignore`). +- Порядок старта (`depends_on`): postgres/valkey/s3/rabbitmq healthy → `migrate` (миграции под advisory lock, теперь 0001–0004) → `s3-init` → `api` ×2 и `worker` → `lb`. +- `FAULT_INJECTION=1` в `.env` включает учебные триггеры сбоев по имени файла `fault-*` (см. ниже). Вне стенда выставить `0`. +- Остановка worker для демонстрации: `scripts/stop-worker.sh` / `.cmd` (= `docker compose stop worker`), запуск — `scripts/start-worker.sh` / `.cmd`. +- Безопасная остановка: `docker compose stop` (`make down`), volumes `pgdata`, `s3data`, `mqdata` сохраняются. Намеренное удаление учебных данных: `docker compose down -v` (необратимо). ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +client → HAProxy (lb, 127.0.0.1:8080) → api ×2 ──┬─ PostgreSQL: attachments, attachment_jobs, job_outbox, + │ job_events, attachment_variants (+ всё из lab4) + ├─ S3 staging/ (оригинал до обработки) + └─ outbox relay ──publish+confirm──► RabbitMQ + exchange attachments + quorum queue attachment.process.v1 + DLX → attachment.process.dlq +worker (отдельный процесс/контейнер) ◄──prefetch=N, manual ack──┘ + ├─ claim/lease в PostgreSQL → читает staging → пишет в S3 attachments// (оригинал) + │ и variants///v1/{thumbnail.jpg,crop.jpg,optimized.webp} → COMMIT результата → удаляет staging → ack + ├─ outbox relay (тот же код, что в API; SKIP LOCKED — несколько relay безопасны) + └─ sweeper: истёкшие lease → queued; retention staging; зависшие pending-загрузки +``` + +**Источник истины** — PostgreSQL: строка `attachment_jobs` определяет, нужно ли что-то делать; сообщение брокера — только «разбудить worker» с `job_id`/`attempt` (`contracts/jobs.schema.json`, без байтов, ключей и URL). Worker берёт доверенные метаданные (ключи, тип, sha256) из БД. + +**Upload, транзакционные границы** (`src/internal/api/attachments.go`, `src/internal/store/jobs.go`): +1. аутентификация → членство (чужой — 404) → filename → Content-Type → `Content-Length` ≤ 10 MiB → дешёвая предпроверка размера очереди (`MAX_ACTIVE_JOBS`, иначе 503 + `Retry-After: 5`) → тело через `MaxBytesReader` → magic bytes + `image.DecodeConfig` (только заголовок: >40 Мпикс → 413). **Полное декодирование и ресайз API больше не делает** — только worker, даже если worker недоступен. +2. `INSERT attachments status='pending'` (невидима в API) → `PUT staging/` (приватный prefix того же bucket, persistent volume S3). +3. **Одна транзакция**: `pg_advisory_xact_lock` + повторный подсчёт активных jobs (строгая граница), `pending→queued`, `INSERT attachment_jobs`, `INSERT job_outbox`, `INSERT job_events('queued')` → COMMIT → `202 queued`. + Значит, в момент 202 устойчиво сохранены: полный проверенный оригинал в S3 staging, метаданные, job и намерение публикации (outbox). Сообщения в брокере в этот момент может ещё не быть — это нормально. +4. Ошибка на 2/3 → 503 и компенсация (удаляем pending-строку, затем staging-объект). Оставшееся (БД недоступна) убирает sweeper через `PENDING_UPLOAD_GRACE=15m`. + +**Transactional outbox** (`src/internal/relay`): relay берёт `job_outbox WHERE published_at IS NULL AND available_at<=now() FOR UPDATE SKIP LOCKED`, публикует persistent-сообщение и **ждёт publisher confirm**, только потом ставит `published_at` в той же транзакции. Relay работает в каждом API (сразу после COMMIT его «пинают») и в worker, плюс опрос раз в секунду. Падение между COMMIT и publish → строку опубликует любой живой relay. Падение после publish до отметки → повторная публикация, т.е. дубль сообщения, который worker обезвреживает (ниже). Недоступность RabbitMQ не ломает upload: 202 продолжает отдаваться, строки копятся в outbox. + +**Broker** — RabbitMQ 4.0.5, quorum-очередь (реплицируемый журнал на volume `mqdata`, durable), `x-delivery-limit=20` и DLX → `attachment.process.dlq` для poison/malformed сообщений. Publisher confirms на стороне relay, manual ack на стороне worker, `prefetch = WORKER_CONCURRENCY`. + +**Worker** (`src/internal/worker/worker.go`), на каждую доставку: +1. Проверка схемы сообщения; некорректное → `nack(requeue=false)` → DLQ. +2. `ClaimJob(job_id, attempt)` под `SELECT … FOR UPDATE`: `ready/failed` → no-op ack; `attempt` в сообщении ≠ текущему → устаревший дубль, ack; `processing` с живым lease → ack (работу держит другой); retry ещё не наступил → ack. Иначе `processing`, `lease_owner`, `lease_until=now()+JOB_LEASE_SECONDS`. **Redelivery** (флаг `redelivered` от RabbitMQ) разрешает перехват lease немедленно: брокер переотдаёт сообщение только после закрытия канала прежнего держателя (crash/kill), поэтому ждать истечения lease не нужно. +3. Чтение оригинала из staging (если staging уже удалён — из конечного ключа), сверка sha256/size с БД, обработка с ограничениями (ниже), запись оригинала и вариантов **по детерминированным ключам** (`(attachment_id, pipeline_version, variant)` → один ключ S3) — повтор перезаписывает те же байты, второго набора не появляется. +4. `CompleteJob` — одна транзакция: `UPDATE job SET ready WHERE job_id AND attempt AND status='processing'`, `INSERT attachment_variants … ON CONFLICT DO NOTHING`, `attachment → ready`. Устаревшая попытка получает 0 строк и ничего не меняет: ready никогда не откатывается. +5. Удаление staging (после COMMIT), `staging_key=NULL` → **ack**. + +**Ошибки и повторы**: `imaging.PermanentError` (`invalid_image`, `image_too_large`, checksum mismatch) → сразу `failed`. Временные (`storage_unavailable`, `processing_timeout`, `lease_expired`, `injected_transient`) → `queued`, `attempt+1`, новая outbox-строка с `available_at = now() + backoff`, backoff экспоненциальный с jitter «equal jitter»: `d = JOB_BACKOFF_BASE·2^(attempt-1)` (cap `JOB_BACKOFF_MAX=60s`), ожидание равномерно в `[d/2, d]`. После `JOB_MAX_ATTEMPTS=5` → `failed / retry_exhausted`. Отложенный повтор реализован через outbox, поэтому не нужен delayed-плагин RabbitMQ и повтор переживает рестарт брокера. Если сама БД недоступна при фиксации — `nack(requeue=true)`. + +**Реестр failed и replay**: `attachment_jobs WHERE status='failed'` + аудит `job_events` (каждая попытка, причина, кто сделал replay). Явный операторский replay: `docker compose exec worker /app/server replay --actor ` — только `failed → queued`, тот же `job_id` (тот же логический эффект), новый бюджет попыток, запись в аудит. Для `ready` replay отказывает (`noop`), для `queued/processing` — тоже; если staging уже удалён retention-ом — `refused`. DLQ брокера содержит только сообщения, которые нельзя разобрать (по ним нет job). + +**Lease / зависший vs долгий worker**: живой worker продлевает lease (heartbeat каждые `lease/3`), но не дольше `JOB_TIMEOUT_SECONDS=60` — потом сам прерывает попытку как `processing_timeout`. Зависший (нет heartbeat) теряет lease через `JOB_LEASE_SECONDS=30`, sweeper (каждые 5 с) переводит job в `queued` с `attempt+1`. Если «оживший» старый worker всё же досчитает, его `CompleteJob` с прежним `attempt` ничего не изменит. Два одновременных исполнителя возможны только в окне сетевого разрыва; они пишут одинаковые байты по одинаковым ключам, а результат фиксирует ровно один. + +**Варианты (pipeline_version=1)**, `src/internal/imaging/imaging.go`: EXIF Orientation (1–8) применяется до ресайза, alpha компонуется на белый, метаданные не копируются ни в один вариант (энкодерам передаются только пиксели). +- `thumbnail`: JPEG q=85, вписан в 256×256 без увеличения, размеры округляются вниз целочисленно (320×200 → 256×160), минимум 1 px. +- `crop`: JPEG q=85, центральный квадрат исходника → 128×128 (маленькие увеличиваются). +- `optimized`: WebP lossy q=80, method=4, вписан в 1280×1280 без увеличения, без EXIF/ICC/XMP. Уменьшение размера маленьких файлов не обещается. +- Ресайз — Catmull-Rom (`x/image/draw`). + +**Лимиты worker**: ≤10 MiB bytes (повторная проверка), ≤40 Мпикс по заголовку до декодирования, общий бюджет памяти декодирования `WORKER_DECODE_BUDGET_BYTES=384 MiB` (взвешенный семафор по оценке `пиксели·8 + 6.5 MiB`; таймаутнувшая попытка держит свою долю до реального завершения декодера), время `JOB_TIMEOUT_SECONDS`, контейнер `768M`/`2 CPU`, `GOMEMLIMIT=600MiB`, `prefetch=concurrency`. + +**Staging cleanup**: удаляется сразу после COMMIT `ready`; у `failed` хранится `STAGING_RETENTION_SECONDS=86400` (для replay), потом sweeper удаляет; брошенные `pending` — через 15 мин. Текущую работу (`queued/processing`) sweeper не выбирает. + +**API-переход 4 → 5**: upload `201 ready` → `202 queued`; `GET /attachments/{id}` показывает `queued/processing/ready/failed`, `variants` только у ready, `error_code` только у failed; `content` (любой variant) для не-ready → 409, отсутствующий variant у ready → 404, неизвестный → 400; к сообщению можно прикрепить queued/processing/ready, failed → 409. + +**Что перенесено из lab4 и что изменено.** Перенесены без изменений пользователи, сессии (Argon2id, Valkey-кэш), чаты, сообщения, идемпотентность, keyset-пагинация, `X-Instance-Id`, HAProxy, S3 (SeaweedFS), privacy bucket, хуки restart/stop-one. Изменено: upload (staging + outbox вместо прямого PUT в конечный ключ), из API убрано полное декодирование, `restart.*` теперь перезапускает и RabbitMQ/worker. **Удалён reconciler lab4** (трек 2 lab4): он считал «осиротевшими» все объекты вне `attachments/`, т.е. удалил бы staging и варианты; его функции по staging/pending взял sweeper worker. Сверка S3↔БД для вариантов в этой работе не реализована (см. ограничения). + +**Соответствие `.env.example`**: `BROKER_URL_FILE`, `WORKER_CONCURRENCY` (= prefetch), `JOB_MAX_ATTEMPTS`, `JOB_TIMEOUT_SECONDS`, `STAGING_RETENTION_SECONDS`, `S3_*`, `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` используются как есть; добавлены `MAX_ACTIVE_JOBS`, `JOB_LEASE_SECONDS`, `JOB_BACKOFF_BASE/MAX`, `WORKER_DECODE_BUDGET_BYTES`, `WORKER_MEMORY/CPUS/GOMEMLIMIT`, `PENDING_UPLOAD_GRACE`, `FAULT_INJECTION`. `S3_ENDPOINT` в примере исправлен на `http://s3:8333` (SeaweedFS). + +| Роль | Service | Образ | Health | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| LB | lb | haproxy:3.0.5-alpine | `:8404/healthz` | — | app; 127.0.0.1:8080 | — | +| API ×2 | api | mtusi-messenger:lab5 | `/health/ready` (только PostgreSQL) | нет (read-only rootfs) | app, data | database_url, cache_url, broker_url, s3_* | +| Worker | worker | mtusi-messenger:lab5 (`worker`) | — (restart unless-stopped) | нет (read-only rootfs) | data | database_url, broker_url, s3_* | +| Broker | rabbitmq | rabbitmq:4.0.5-management-alpine | `check_port_connectivity` | mqdata | data, не публикуется | rabbitmq_auth.conf | +| БД | postgres | postgres:17.2-alpine3.21 | pg_isready | pgdata | data | pg_password | +| Сессии | valkey | valkey/valkey:8.0.2-alpine3.21 | ping | — | data | valkey_auth | +| S3 | s3 | chrislusf/seaweedfs:3.80 | `:8333/healthz` | s3data | data | s3_identities.json | +| Bootstrap | migrate, s3-init | mtusi-messenger:lab5 | one-shot | — | data | database_url / s3_admin_* | + +`/health/ready` зависит только от PostgreSQL: недоступность брокера/worker/S3 не снимает API с балансировки (текстовый мессенджер работает, upload при живом S3 продолжает отдавать 202 через outbox). + +**Учебные триггеры сбоев** (`FAULT_INJECTION=1`, срабатывают один раз на файл по префиксу имени): `fault-api-after-commit` (API `os.Exit` после COMMIT до publish/ответа), `fault-worker-after-effect` (после записи S3 до COMMIT), `fault-worker-before-ack` (после COMMIT до ack), `fault-transient` (попытки 1–2 падают), `fault-flaky` (падают все попытки до replay), `fault-hang` (попытка 1 «замерзает» на 3×lease без heartbeat). ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Окружение: Windows 11, Docker Desktop 29.7.2 (WSL2); лимиты: API 1 CPU/512M ×2, worker 2 CPU/768M (`GOMEMLIMIT=600MiB`), RabbitMQ 1 CPU/512M, S3 1 CPU/1G, PostgreSQL 2 CPU/1G. Дата прогона — 2026-11-11/12. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +### Уровень 3 + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл | +| --- | --- | --- | --- | +| Публичный smoke (202 → ready, оригинал, JPEG thumbnail 256×160) | `make test` | `PASS: lab5 public API scenario` | `evidence/smoke.txt` | +| Минимум lab4 сохранён, сохранность после рестарта всех хранилищ | `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` (stop api/worker/lb, restart postgres/valkey/s3/rabbitmq) | `PASS: persistence preserved confirmed data and existing session` | `evidence/persistence.txt` | +| Отказ одной API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd` | 117 записей, 0 ошибок, восстановление 3.97 с, `PASS` | `evidence/failover.txt` | +| 202 только после устойчивого приёма; остановка API после ответа не теряет работу | `lab5ops.py worker-down`: worker остановлен → upload → `restart api` + `restart rabbitmq` → `start worker` | `202 queued`; сообщение с queued-вложением 201; content 409; текстовые сообщения 201; в очереди 1 сообщение до и **после** рестарта API+broker; после старта worker ready за 0.34 с | `evidence/worker-down.json` | +| Отдельные broker и worker, durable + at-least-once, ack после эффекта | `compose.yaml` (rabbitmq quorum, worker), сценарии `effect-crash`/`ack-crash` | Сообщение пережило рестарт брокера с volume; повтор доставки после убийства worker завершил ту же job | `evidence/worker-down.json`, `evidence/effect-crash.json` | +| Недоступность worker не вызывает синхронный resize | API не декодирует изображения вообще (`inspectImage` читает только заголовок) | upload при остановленном worker 202, статус `queued` до запуска worker | `evidence/worker-down.json` | +| Состояния queued/processing/ready/failed; thumbnail по контракту; original неизменен | `worker-down`, `invalid`, `replay` | seen `processing→ready`, `failed (invalid_image / retry_exhausted)`; thumbnail реальный JPEG 256×160; sha256 оригинала совпадает | `evidence/*.json` (`ready_check`) | +| Ограниченный retry + финальный failed с безопасным кодом | `lab5ops.py replay` (`JOB_MAX_ATTEMPTS=5`) | 5 попыток → `failed`, `error_code=retry_exhausted`, без stack trace | `evidence/replay.json` | +| Повтор одной job не создаёт второго набора | `effect-crash`, `ack-crash`, `hang` | везде `variant_rows=3`, в S3 `variants///` ровно 3 объекта | `evidence/effect-crash.json`, `ack-crash.json`, `hang.json` | + +### Уровень 4 + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл | +| --- | --- | --- | --- | +| crop 128×128 и optimized WebP; параметры описаны | любой ready (`worker-down`), `make unit` | `thumbnail image/jpeg 256x160`, `crop image/jpeg 128x128`, `optimized image/webp 320x200`; unit-тесты: Fit, EXIF orientation 6 (320×200 → thumbnail 160×256), alpha → белый | `evidence/worker-down.json`, `src/internal/imaging/imaging_test.go` | +| Невалидные/огромные изображения ограничены по bytes, пикселям, времени, памяти | `lab5ops.py invalid`, `lab5ops.py limits` | битый PNG с верным заголовком: 202 → `failed invalid_image` за **1** попытку (постоянная ошибка не ретраится); API: pixel bomb 20000×20000 → 413, не-изображение → 415, >10 MiB → 413, `../` в имени → 400; 6000×6000 (36 Мпикс) → ready за 5.2 с, пик памяти worker 491 MiB при лимите 768 MiB | `evidence/invalid.json`, `evidence/limits.json` | +| Разрыв PostgreSQL ↔ broker закрыт outbox; crash между COMMIT и publish | `lab5ops.py api-crash` (`fault-api-after-commit`) | клиент получил 502 (API-процесс умер после COMMIT), API-контейнер перезапущен (`RestartCount 0→1`), строку outbox опубликовал другой relay, job `ready` без ручных действий; события `queued → started → ready` | `evidence/api-crash.json` | +| Backoff с jitter, постоянные vs временные, DLQ/реестр failed, безопасный replay | `lab5ops.py retry`, `replay`, `dlq` | паузы перед попытками 2 и 3: 0.94 с и 1.81 с (диапазоны [0.5,1] и [1,2]); `fault-flaky`: 5 попыток → `failed retry_exhausted`; content 409, прикрепление к сообщению 409; `replay` → `{"result":"replayed","attempt":6}` → ready; повторный replay → `noop: job already ready`; malformed сообщение → `attachment.process.dlq` 0→1 | `evidence/retry.json`, `replay.json`, `dlq.json` | +| Остановка worker после записи объекта, до ack — повтор завершает тот же результат | `lab5ops.py effect-crash` (после S3 до COMMIT), `ack-crash` (после COMMIT до ack) | effect-crash: события `started → taken_over → ready`, worker `RestartCount 0→1`, 3 варианта; ack-crash: `ready` один раз, после рестарта redelivery → лог `job delivery is a no-op result=done redelivered=true` | `evidence/effect-crash.json`, `evidence/ack-crash.json` | +| Timeout/lease для зависшего processing | `lab5ops.py hang` | попытка 1 зависла без heartbeat → через 31 с sweeper: `retry_scheduled lease_expired` → попытка 2 → ready; «оживший» старый исполнитель: `stale attempt finished after takeover; result already owned by newer attempt`, вариантов по-прежнему 3 | `evidence/hang.json` | +| Ограниченная конкуренция/prefetch | `WORKER_CONCURRENCY` = число goroutine = `basic.qos prefetch`; бюджет декодирования; `MAX_ACTIVE_JOBS` | см. трек 1: при `MAX_ACTIVE_JOBS=20` принято ровно 20 из 80 | `evidence/track1/*.json` | +| Очистка staging после безопасной фиксации | все сценарии ready | `staging_removed=true`, в S3 `staging/` 0 объектов, конечный оригинал 1; у failed staging сохраняется для replay | `evidence/worker-down.json` (`s3`) | + +### Трек 1 — очередь под перегрузкой + +**Гипотеза.** Pipeline изображений CPU-bound: увеличение `WORKER_CONCURRENCY` (= prefetch) повышает throughput почти линейно, пока число параллельных задач не упрётся в CPU-лимит worker (2 CPU), а для больших изображений — в бюджет памяти декодирования (384 MiB ≈ 2 задачи по 20 Мпикс). Дальше рост concurrency не даёт throughput, но увеличивает RSS. Без ограничения принятой работы очередь и время до ready растут неограниченно; с `MAX_ACTIVE_JOBS` лишняя работа отклоняется сразу (503 + `Retry-After`), а принятая гарантированно дренируется. + +**Методика** (`scripts/track1/run_track1.py`, `make track1`). Для каждой настройки worker пересоздаётся с `WORKER_CONCURRENCY ∈ {1,2,4}` (те же лимиты 2 CPU / 768M / `GOMEMLIMIT=600MiB`, бюджет декодирования 384 MiB), для каждого размера — новый чат и **открытый burst** N загрузок 8 клиентскими потоками без ожидания обработки: +- S = `tests/fixtures/sample.png` 320×200 (112 КБ), N=120; +- M = 2000×1500 PNG с шумовыми строками (1.17 МБ), N=40; +- L = 5000×4000 PNG-градиент (66 КБ на диске, 20 Мпикс после декодирования), N=16. + +Метрики: время до ready каждой job = `finished_at − created_at` строки `attachment_jobs` (часы сервера, без клиентского polling); throughput = ready / (последний finished − первый created); queue lag — выборка каждые 0.5 с: активные jobs (queued+processing), `messages_ready` в RabbitMQ (`rabbitmqctl list_queues`), возраст самой старой незавершённой job; RSS — пик `docker stats` контейнера worker. Прогрев: стенд уже работал, по одному прогону на ячейку (ограничение — см. ниже). Raw: `evidence/track1/matrix.json` (с таймлайном lag), `evidence/track1/backpressure.json`. + +| Размер | conc/prefetch | принято / ready / failed | throughput, job/s | p50 / p95 до ready, с | max queue (DB / MQ ready) | max возраст старейшей job, с | пик RSS worker, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | +| S 320×200 | 1 | 120 / 120 / 0 | 9.38 | 6.3 / 11.3 | 51 / 112 | 10.9 | 59 | +| S 320×200 | 2 | 120 / 120 / 0 | 16.25 | 3.6 / 6.2 | 63 / 64 | 6.4 | 23 | +| S 320×200 | 4 | 120 / 120 / 0 | **23.27** | **2.3 / 3.9** | 61 / 32 | 0.5 | 29 | +| M 2000×1500 | 1 | 40 / 40 / 0 | 0.56 | 36.8 / 68.1 | 36 / 38 | 69.3 | 165 | +| M 2000×1500 | 2 | 40 / 40 / 0 | **1.11** | **19.2 / 35.0** | 36 / 34 | 33.8 | 328 | +| M 2000×1500 | 4 | 40 / 40 / 0 | 1.09 | 21.1 / 35.8 | 36 / 36 | 32.1 | 513 | +| L 5000×4000 | 1 | 16 / 16 / 0 | 0.37 | 24.2 / 42.8 | 16 / 14 | 38.3 | 601 | +| L 5000×4000 | 2 | 16 / 16 / 0 | **0.72** | **13.6 / 22.0** | 16 / 14 | 21.5 | 585 | +| L 5000×4000 | 4 | 16 / 16 / 0 | 0.73 | 13.7 / 21.7 | 16 / 12 | 19.6 | 593 | + +**Backpressure и восстановление** (`backpressure`): API с `MAX_ACTIVE_JOBS=20`, worker concurrency 1, burst 80 M-изображений за 1.1 с → **принято ровно 20, 60 × 503 `unavailable` с `Retry-After: 5`**, ни одного другого кода; максимум активных jobs по выборкам 18 ≤ 20. Все 20 принятых дошли до ready (0 failed, 0 retry), очередь опустела за 37.5 с (throughput 0.56 job/s — как в матрице для conc=1), и **сразу после перегрузки** новый upload стал ready за 0.34 с — система вернулась к idle-латентности без ручных действий. Первая версия ограничения (только предпроверка до чтения тела) пропустила 25 из 80 при пределе 20 (гонка check-then-insert между параллельными загрузками) — после этого добавлена повторная проверка в транзакции под `pg_advisory_xact_lock`, граница стала строгой. + +**Выводы.** +1. Для мелких изображений накладные расходы (БД-транзакции, S3 PUT ×4, ack) доминируют, а не CPU: throughput растёт 9.4 → 16.3 → 23.3 job/s (×2.5 при ×4 concurrency), p95 падает 11.3 → 3.9 с, RSS остаётся < 60 MiB. Здесь concurrency 4 однозначно лучше. +2. Для M и L рост останавливается на concurrency 2 (= 2 CPU лимита): 1→2 даёт ×2 throughput и ×2 меньше p95, 2→4 — 0 % (1.11 → 1.09 и 0.72 → 0.73), зато RSS для M растёт 328 → 513 MiB. Для L concurrency 4 фактически не включается: бюджет декодирования 384 MiB пропускает только 2 задачи по 20 Мпикс одновременно, поэтому RSS держится ~590 MiB (у потолка `GOMEMLIMIT`), а OOM нет ни в одном прогоне. Гипотеза подтверждена: оптимум concurrency ≈ число CPU для тяжёлых изображений, больше — только трата памяти. +3. Компромисс latency/ресурсы: при фиксированном CPU latency в очереди определяется законом Литтла — время до ready ≈ длина очереди / throughput; при неограниченном приёме p95 растёт линейно с burst (M, conc=1: 68 с для 40 файлов). Ограничение принятой работы делает худшую задержку предсказуемой (≈ `MAX_ACTIVE_JOBS` / throughput: 20 / 0.56 ≈ 36 с, наблюдали max 35 с) ценой явного 503 для клиента. +4. Выбранная конфигурация по умолчанию: `WORKER_CONCURRENCY=2` (=CPU), бюджет 384 MiB как предохранитель памяти, `MAX_ACTIVE_JOBS=500` (для демонстрации backpressure — 20). + +**Ограничения измерения.** По одному прогону на ячейку (разброс не оценён, повтор — `make track1`); один хост Windows/WSL2, клиент и стенд делят CPU; RSS — выборки `docker stats` раз в ~1–2 с, кратковременные пики могут быть пропущены; L-изображение синтетическое и хорошо сжимается (нагрузка — декодирование/ресайз, а не передача байтов); размер M-файлов искусственный (шум каждые 8 строк). Выводы относятся к этим лимитам и профилю, а не к production. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| worker остановлен, upload, рестарт API и RabbitMQ (volumes сохранены) | 202, задача не теряется | 202 queued; сообщение в quorum-очереди до и после рестарта; ready через 0.34 с после старта worker | потерь нет | +| API умирает между COMMIT и publish | outbox опубликует | клиент 502, job ready через ~0.2 с (relay другой реплики/worker) | потерь нет; клиент не знает id — повтор upload создаст второе вложение (контракт это допускает) | +| worker умирает после записи S3 до COMMIT | redelivery, перехват lease, тот же результат | `taken_over → ready`, 3 варианта | ~1 с (рестарт контейнера), дублей нет | +| worker умирает после COMMIT до ack | redelivery — no-op | `result=done redelivered=true`, ack | дублей нет | +| worker «завис» | lease истекает, новая попытка | ready через 31 с (lease 30 с) | старая попытка не перезаписала результат | +| временные ошибки | backoff с jitter | 0.94 с, 1.81 с | ready на 3-й попытке | +| постоянная ошибка (битый PNG) | failed без ретраев | `invalid_image` за 1 попытку | — | +| malformed сообщение в брокере | DLQ | DLQ 0→1 | — | +| перегрузка (80 загрузок при лимите 20) | 503 + Retry-After, работа ограничена | 20 принято, 60 × 503 `Retry-After: 5`, после дренажа новый файл ready за 0.34 с | принятые 20 все ready | +| рестарт всех хранилищ (`restart.cmd`) | данные, сессия, файл сохранены | `make test-persistence` PASS | — | +| остановка одной API-реплики | запись продолжается | 0 ошибок, 3.97 с до первой записи после hook | — | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один Docker-host: RabbitMQ — одна нода (quorum-очередь из одного члена), PostgreSQL и S3 — по одному экземпляру. Выдерживаются рестарты процессов с сохранёнными volumes, а не потеря диска/хоста. +- At-least-once + идемпотентный эффект ≠ exactly-once: один и тот же S3-ключ может быть записан несколько раз (одинаковыми байтами), а дубль сообщения — получен; «ровно один раз» гарантируется только для зафиксированного результата в PostgreSQL. +- При crash между COMMIT и ответом клиент не узнаёт `attachment_id`; без расширения контракта (Idempotency-Key для upload) повтор создаст второе вложение. +- Таймаут не прерывает декодер Go насильно: зависшая попытка освобождает свою долю бюджета памяти только по фактическому завершению декодирования; защиту от бесконечного декодера даёт лимит контейнера. +- Сверка S3 ↔ БД для вариантов (reconciler lab4) не перенесена; объекты от устаревших попыток всегда совпадают с зафиксированными по ключу, но «сиротские» объекты удалённых вложений не собираются. +- Fault injection по имени файла — учебный механизм, выключается `FAULT_INJECTION=0`. +- Worker использует ту же S3-идентичность, что и API; следующий шаг — отдельная идентичность с правами только на нужные prefix-ы. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..18fa9697 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,296 @@ +name: mtusi-lab5 + +x-api-env: &api-env + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_STORE: ${SESSION_STORE:-valkey} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_CACHE_MAX_TTL_SECONDS: ${SESSION_CACHE_MAX_TTL_SECONDS:-60} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + SHUTDOWN_DRAIN_MS: ${SHUTDOWN_DRAIN_MS:-3000} + DB_MAX_CONNS: ${DB_MAX_CONNS:-10} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_TIMEOUT: ${S3_TIMEOUT:-15s} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-8} + UPLOAD_TIMEOUT: ${UPLOAD_TIMEOUT:-60s} + GOMEMLIMIT: ${API_GOMEMLIMIT:-400MiB} + BROKER_URL_FILE: /run/secrets/broker_url + MAX_ACTIVE_JOBS: ${MAX_ACTIVE_JOBS:-500} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + FAULT_INJECTION: ${FAULT_INJECTION:-0} + +x-worker-env: &worker-env + DATABASE_URL_FILE: /run/secrets/database_url + DB_MAX_CONNS: "${WORKER_DB_MAX_CONNS:-8}" + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_DECODE_BUDGET_BYTES: ${WORKER_DECODE_BUDGET_BYTES:-402653184} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-30} + JOB_BACKOFF_BASE: ${JOB_BACKOFF_BASE:-1s} + JOB_BACKOFF_MAX: ${JOB_BACKOFF_MAX:-60s} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + PENDING_UPLOAD_GRACE: ${PENDING_UPLOAD_GRACE:-15m} + FAULT_INJECTION: ${FAULT_INJECTION:-0} + GOMEMLIMIT: ${WORKER_GOMEMLIMIT:-600MiB} + +services: + postgres: + image: postgres:17.2-alpine3.21 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/pg_password + command: ["postgres", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"] + secrets: [pg_password] + volumes: + - pgdata:/var/lib/postgresql/data + networks: [data] + shm_size: 256mb + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h 127.0.0.1"] + interval: 2s + timeout: 3s + retries: 30 + deploy: + resources: + limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1G}"} + stop_grace_period: 30s + restart: unless-stopped + + valkey: + image: valkey/valkey:8.0.2-alpine3.21 + command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"] + volumes: + - ./infra/valkey/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro + secrets: [valkey_auth] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "valkey-cli ping | grep -qE 'PONG|NOAUTH'"] + interval: 2s + timeout: 3s + retries: 30 + deploy: + resources: + limits: {cpus: "0.5", memory: 128M} + restart: unless-stopped + + s3: + image: chrislusf/seaweedfs:3.80 + command: + - server + - -dir=/data + - -s3 + - -s3.port=8333 + - -s3.config=/run/secrets/s3_identities + - -master.volumeSizeLimitMB=64 + - -volume.max=100 + secrets: [s3_identities] + volumes: + - s3data:/data + networks: [data] + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"] + interval: 2s + timeout: 3s + start_period: 60s + retries: 60 + deploy: + resources: + limits: {cpus: "1.0", memory: "${S3_MEMORY:-1G}"} + stop_grace_period: 20s + restart: unless-stopped + + s3-init: + image: mtusi-messenger:lab5 + pull_policy: never + command: ["s3-init"] + healthcheck: + disable: true + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: [s3_admin_access_key_id, s3_admin_secret_access_key] + networks: [data] + depends_on: + s3: + condition: service_healthy + migrate: + condition: service_completed_successfully + deploy: + resources: + limits: {cpus: "0.5", memory: 64M} + restart: "no" + + migrate: + build: + context: . + args: + APP_BUILD: ${APP_BUILD:-dev} + image: mtusi-messenger:lab5 + command: ["migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [data] + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: {cpus: "0.5", memory: 128M} + restart: "no" + + api: + image: mtusi-messenger:lab5 + pull_policy: never + environment: *api-env + read_only: true + tmpfs: + - /tmp:size=1m + secrets: [database_url, cache_url, broker_url, s3_access_key_id, s3_secret_access_key] + networks: [app, data] + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + postgres: + condition: service_healthy + valkey: + condition: service_healthy + rabbitmq: + condition: service_started + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"] + interval: 2s + timeout: 2s + start_period: 5s + retries: 3 + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-512M}"} + stop_grace_period: 15s + restart: unless-stopped + + rabbitmq: + image: rabbitmq:4.0.5-management-alpine + hostname: rabbitmq + secrets: + - source: rabbitmq_auth + target: /etc/rabbitmq/conf.d/20-auth.conf + volumes: + - mqdata:/var/lib/rabbitmq + networks: [data] + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"] + interval: 5s + timeout: 10s + start_period: 20s + retries: 30 + deploy: + resources: + limits: {cpus: "1.0", memory: 512M} + stop_grace_period: 30s + restart: unless-stopped + + worker: + image: mtusi-messenger:lab5 + pull_policy: never + command: ["worker"] + environment: *worker-env + read_only: true + tmpfs: + - /tmp:size=1m + secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] + networks: [data] + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + deploy: + replicas: ${WORKER_REPLICAS:-1} + resources: + limits: {cpus: "${WORKER_CPUS:-2.0}", memory: "${WORKER_MEMORY:-768M}"} + stop_grace_period: 30s + restart: unless-stopped + + lb: + image: haproxy:3.0.5-alpine + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + - "[::1]:${APP_PORT:-8080}:8080" + networks: [app] + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"] + interval: 2s + timeout: 2s + retries: 15 + deploy: + resources: + limits: {cpus: "${LB_CPUS:-1.0}", memory: 128M} + restart: unless-stopped + +networks: + app: {} + data: {} + +volumes: + pgdata: {} + s3data: {} + mqdata: {} + +secrets: + pg_password: + file: ./secrets/pg_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + valkey_auth: + file: ./secrets/valkey_auth.conf + s3_identities: + file: ./secrets/s3_identities.json + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + rabbitmq_auth: + file: ./secrets/rabbitmq_auth.conf + broker_url: + file: ./secrets/broker_url diff --git a/evidence/ack-crash.json b/evidence/ack-crash.json new file mode 100644 index 00000000..2538595d --- /dev/null +++ b/evidence/ack-crash.json @@ -0,0 +1,72 @@ +{ + "scenario": "ack-crash", + "result": "PASS", + "started_at": "2026-11-11T23:24:36Z", + "fault": "worker os.Exit after result COMMIT + staging cleanup, before ack", + "attachment_id": "fc38c3d8-b1b7-440f-b6c5-9b315979731b", + "upload": 202, + "worker_restart_counts_before": { + "d3259669fad0": 1 + }, + "worker_restart_counts_after": { + "d3259669fad0": 2 + }, + "statuses_seen": [ + "processing", + "ready" + ], + "events": [ + { + "attempt": "1", + "event": "queued", + "error_code": "", + "at": "23:24:38.431" + }, + { + "attempt": "1", + "event": "started", + "error_code": "", + "at": "23:24:38.460" + }, + { + "attempt": "1", + "event": "ready", + "error_code": "", + "at": "23:24:38.529" + } + ], + "job_after": { + "job_id": "4b9eb573-b6b4-4f6c-bec9-0d73147bab15", + "status": "ready", + "attempt": 1, + "max_attempts": 5, + "error_code": "", + "variant_rows": 3, + "outbox_rows": 1, + "outbox_unpublished": 0, + "staging_removed": true + }, + "s3": { + "variants_objects": 3, + "final_original": 1, + "staging_objects": 0 + }, + "redelivery_noop_log": [ + { + "result": "done", + "redelivered": true + } + ], + "ready_check": { + "variants": { + "thumbnail": "image/jpeg 256x160 5620B", + "crop": "image/jpeg 128x128 2440B", + "optimized": "image/webp 320x200 2462B" + }, + "thumbnail_real_jpeg_size": [ + 256, + 160 + ], + "original_sha256_ok": true + } +} diff --git a/evidence/api-crash.json b/evidence/api-crash.json new file mode 100644 index 00000000..a681d8af --- /dev/null +++ b/evidence/api-crash.json @@ -0,0 +1,73 @@ +{ + "scenario": "api-crash", + "result": "PASS", + "started_at": "2026-11-11T23:24:10Z", + "fault": "API os.Exit right after COMMIT(queued+job+outbox), before publish and before HTTP response", + "attachment_id": "4e18930b-bd0f-4db5-a033-35fbd74c93a1", + "client_view": "client saw failure: -a15d-9d5fe0759a8c/attachments: expected [202], received 502", + "job_right_after_crash": { + "job_id": "8dadbdde-897c-4acb-ae12-b7565b03dbb4", + "status": "ready", + "attempt": 1, + "max_attempts": 5, + "error_code": "", + "variant_rows": 3, + "outbox_rows": 1, + "outbox_unpublished": 0, + "staging_removed": true + }, + "api_restart_counts_before": { + "78c8984ff002": 0, + "4c38672b42d2": 0 + }, + "api_restart_counts_after": { + "78c8984ff002": 1, + "4c38672b42d2": 0 + }, + "statuses_seen": [ + "ready" + ], + "events": [ + { + "attempt": "1", + "event": "queued", + "error_code": "", + "at": "23:24:12.312" + }, + { + "attempt": "1", + "event": "started", + "error_code": "", + "at": "23:24:12.463" + }, + { + "attempt": "1", + "event": "ready", + "error_code": "", + "at": "23:24:12.538" + } + ], + "job_after": { + "job_id": "8dadbdde-897c-4acb-ae12-b7565b03dbb4", + "status": "ready", + "attempt": 1, + "max_attempts": 5, + "error_code": "", + "variant_rows": 3, + "outbox_rows": 1, + "outbox_unpublished": 0, + "staging_removed": true + }, + "ready_check": { + "variants": { + "thumbnail": "image/jpeg 256x160 5620B", + "crop": "image/jpeg 128x128 2440B", + "optimized": "image/webp 320x200 2462B" + }, + "thumbnail_real_jpeg_size": [ + 256, + 160 + ], + "original_sha256_ok": true + } +} diff --git a/evidence/dlq.json b/evidence/dlq.json new file mode 100644 index 00000000..c677f942 --- /dev/null +++ b/evidence/dlq.json @@ -0,0 +1,21 @@ +{ + "scenario": "dlq", + "result": "PASS", + "started_at": "2026-11-11T23:25:29Z", + "published": { + "routed": true + }, + "dlq_before": { + "ready": 0, + "unacked": 0, + "consumers": 0 + }, + "dlq_after": { + "ready": 1, + "unacked": 0, + "consumers": 0 + }, + "worker_log": [ + "malformed job message -> DLQ" + ] +} diff --git a/evidence/effect-crash.json b/evidence/effect-crash.json new file mode 100644 index 00000000..eb80c2c6 --- /dev/null +++ b/evidence/effect-crash.json @@ -0,0 +1,74 @@ +{ + "scenario": "effect-crash", + "result": "PASS", + "started_at": "2026-11-11T23:24:19Z", + "fault": "worker os.Exit after writing original+variants to S3, before result COMMIT and ack", + "attachment_id": "4e2f8752-7072-4dc0-ae24-eccb7f6d25ff", + "upload": 202, + "worker_restart_counts_before": { + "d3259669fad0": 0 + }, + "worker_restart_counts_after": { + "d3259669fad0": 1 + }, + "statuses_seen": [ + "queued", + "processing", + "ready" + ], + "events": [ + { + "attempt": "1", + "event": "queued", + "error_code": "", + "at": "23:24:21.431" + }, + { + "attempt": "1", + "event": "started", + "error_code": "", + "at": "23:24:21.482" + }, + { + "attempt": "1", + "event": "taken_over", + "error_code": "", + "at": "23:24:22.154" + }, + { + "attempt": "1", + "event": "ready", + "error_code": "", + "at": "23:24:22.573" + } + ], + "job_after": { + "job_id": "f945f7fb-dd8f-461b-b3e5-7cae6db38fcc", + "status": "ready", + "attempt": 1, + "max_attempts": 5, + "error_code": "", + "variant_rows": 3, + "outbox_rows": 1, + "outbox_unpublished": 0, + "staging_removed": true + }, + "s3": { + "variants_objects": 3, + "final_original": 1, + "staging_objects": 0 + }, + "redelivery_noop_log": [], + "ready_check": { + "variants": { + "thumbnail": "image/jpeg 256x160 5620B", + "crop": "image/jpeg 128x128 2440B", + "optimized": "image/webp 320x200 2462B" + }, + "thumbnail_real_jpeg_size": [ + 256, + 160 + ], + "original_sha256_ok": true + } +} diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..b3cc9dd9 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 117, "failed_attempts": 0, "first_recovery_seconds": 3.972} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/hang.json b/evidence/hang.json new file mode 100644 index 00000000..a6dc3de2 --- /dev/null +++ b/evidence/hang.json @@ -0,0 +1,77 @@ +{ + "scenario": "hang", + "result": "PASS", + "started_at": "2026-11-11T23:25:57Z", + "fault": "attempt 1 freezes for 3x lease without heartbeat (simulated hung worker)", + "attachment_id": "9eb5e502-1491-4ae3-93ae-5031997a921d", + "seconds_to_ready": 31.6, + "statuses_seen": [ + "queued", + "processing", + "queued", + "ready" + ], + "events": [ + { + "attempt": "1", + "event": "queued", + "error_code": "", + "at": "23:25:57.874" + }, + { + "attempt": "1", + "event": "started", + "error_code": "", + "at": "23:25:57.929" + }, + { + "attempt": "1", + "event": "retry_scheduled", + "error_code": "lease_expired", + "at": "23:26:29.068" + }, + { + "attempt": "2", + "event": "started", + "error_code": "", + "at": "23:26:29.319" + }, + { + "attempt": "2", + "event": "ready", + "error_code": "", + "at": "23:26:29.391" + } + ], + "job_after": { + "job_id": "f955fe0d-3b17-42bf-8df8-aae785e1ae68", + "status": "ready", + "attempt": 2, + "max_attempts": 5, + "error_code": "", + "variant_rows": 3, + "outbox_rows": 2, + "outbox_unpublished": 0, + "staging_removed": true + }, + "s3": { + "variants_objects": 3, + "final_original": 1, + "staging_objects": 0 + }, + "stale_attempt_logs": [ + "stale attempt finished after takeover; result already owned by newer attempt" + ], + "ready_check": { + "variants": { + "thumbnail": "image/jpeg 256x160 5620B", + "crop": "image/jpeg 128x128 2440B", + "optimized": "image/webp 320x200 2462B" + }, + "thumbnail_real_jpeg_size": [ + 256, + 160 + ], + "original_sha256_ok": true + } +} diff --git a/evidence/invalid.json b/evidence/invalid.json new file mode 100644 index 00000000..257b8029 --- /dev/null +++ b/evidence/invalid.json @@ -0,0 +1,50 @@ +{ + "scenario": "invalid", + "result": "PASS", + "started_at": "2026-11-11T23:25:26Z", + "corrupt_png": { + "upload": 202, + "statuses_seen": [ + "processing", + "failed" + ], + "error_code": "invalid_image", + "events": [ + { + "attempt": "1", + "event": "queued", + "error_code": "", + "at": "23:25:26.980" + }, + { + "attempt": "1", + "event": "started", + "error_code": "", + "at": "23:25:27.016" + }, + { + "attempt": "1", + "event": "failed", + "error_code": "invalid_image", + "at": "23:25:27.027" + } + ], + "job": { + "job_id": "7ead7c8c-0dd7-4b4a-8c2a-807323b1673a", + "status": "failed", + "attempt": 1, + "max_attempts": 5, + "error_code": "invalid_image", + "variant_rows": 0, + "outbox_rows": 1, + "outbox_unpublished": 0, + "staging_removed": false + } + }, + "api_rejects_before_queue": { + "pixel_bomb_20000x20000": 413, + "not_an_image": 415, + "over_10MiB": 413, + "path_in_filename": 400 + } +} diff --git a/evidence/limits.json b/evidence/limits.json new file mode 100644 index 00000000..3cb0389f --- /dev/null +++ b/evidence/limits.json @@ -0,0 +1,29 @@ +{ + "scenario": "limits", + "result": "PASS", + "started_at": "2026-11-11T23:25:46Z", + "image": "6000x6000 PNG (36 MPx, 116950 bytes)", + "seconds_to_ready": 5.2, + "worker_peak_mem_mib": 490.6, + "worker_mem_limit_mib": 768, + "variants": { + "thumbnail": "256x256", + "crop": "128x128", + "optimized": "1280x1280" + }, + "statuses_seen": [ + "processing", + "ready" + ], + "job": { + "job_id": "a0763969-5958-4662-b71a-5e861560a79b", + "status": "ready", + "attempt": 1, + "max_attempts": 5, + "error_code": "", + "variant_rows": 3, + "outbox_rows": 1, + "outbox_unpublished": 0, + "staging_removed": true + } +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..749fd29b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/replay.json b/evidence/replay.json new file mode 100644 index 00000000..61a4f34b --- /dev/null +++ b/evidence/replay.json @@ -0,0 +1,154 @@ +{ + "scenario": "replay", + "result": "PASS", + "started_at": "2026-11-11T23:28:58Z", + "fault": "transient error on every attempt <= JOB_MAX_ATTEMPTS", + "attachment_id": "adcb5aa2-0565-4bc7-b88c-c44cab442533", + "statuses_seen": [ + "queued", + "processing", + "queued", + "failed" + ], + "failed_metadata": { + "status": "failed", + "error_code": "retry_exhausted", + "variants": [] + }, + "content_when_failed": 409, + "attach_failed_to_message": 409, + "job_failed": { + "job_id": "089d500a-3ea5-43bb-a0fd-691bf514537d", + "status": "failed", + "attempt": 5, + "max_attempts": 5, + "error_code": "retry_exhausted", + "variant_rows": 0, + "outbox_rows": 5, + "outbox_unpublished": 0, + "staging_removed": false + }, + "dlq_registry_query": "1", + "replay_1": { + "attempt": 6, + "job_id": "089d500a-3ea5-43bb-a0fd-691bf514537d", + "result": "replayed" + }, + "replay_2_after_ready": { + "result": "noop", + "reason": "job already ready; replay refused" + }, + "statuses_after_replay": [ + "processing", + "ready" + ], + "events": [ + { + "attempt": "1", + "event": "queued", + "error_code": "", + "at": "23:28:59.257" + }, + { + "attempt": "1", + "event": "started", + "error_code": "", + "at": "23:28:59.318" + }, + { + "attempt": "1", + "event": "retry_scheduled", + "error_code": "injected_transient", + "at": "23:28:59.324" + }, + { + "attempt": "2", + "event": "started", + "error_code": "", + "at": "23:29:00.268" + }, + { + "attempt": "2", + "event": "retry_scheduled", + "error_code": "injected_transient", + "at": "23:29:00.277" + }, + { + "attempt": "3", + "event": "started", + "error_code": "", + "at": "23:29:02.077" + }, + { + "attempt": "3", + "event": "retry_scheduled", + "error_code": "injected_transient", + "at": "23:29:02.085" + }, + { + "attempt": "4", + "event": "started", + "error_code": "", + "at": "23:29:05.701" + }, + { + "attempt": "4", + "event": "retry_scheduled", + "error_code": "injected_transient", + "at": "23:29:05.708" + }, + { + "attempt": "5", + "event": "started", + "error_code": "", + "at": "23:29:11.048" + }, + { + "attempt": "5", + "event": "failed", + "error_code": "retry_exhausted", + "at": "23:29:11.060" + }, + { + "attempt": "6", + "event": "replay", + "error_code": "", + "at": "23:29:13.048" + }, + { + "attempt": "6", + "event": "started", + "error_code": "", + "at": "23:29:13.264" + }, + { + "attempt": "6", + "event": "ready", + "error_code": "", + "at": "23:29:13.784" + } + ], + "job_after": { + "job_id": "089d500a-3ea5-43bb-a0fd-691bf514537d", + "status": "ready", + "attempt": 6, + "max_attempts": 10, + "error_code": "", + "variant_rows": 3, + "outbox_rows": 6, + "outbox_unpublished": 0, + "staging_removed": true + }, + "ready_check": { + "variants": { + "thumbnail": "image/jpeg 256x160 5620B", + "crop": "image/jpeg 128x128 2440B", + "optimized": "image/webp 320x200 2462B" + }, + "thumbnail_real_jpeg_size": [ + 256, + 160 + ], + "original_sha256_ok": true + } +} diff --git a/evidence/retry.json b/evidence/retry.json new file mode 100644 index 00000000..038dbd8d --- /dev/null +++ b/evidence/retry.json @@ -0,0 +1,87 @@ +{ + "scenario": "retry", + "result": "PASS", + "started_at": "2026-11-11T23:24:52Z", + "fault": "attempts 1 and 2 raise a transient error (injected_transient)", + "attachment_id": "1eadb246-c3b9-4152-b21e-1b5ec2de9033", + "statuses_seen": [ + "queued", + "processing", + "ready" + ], + "events": [ + { + "attempt": "1", + "event": "queued", + "error_code": "", + "at": "23:24:53.308" + }, + { + "attempt": "1", + "event": "started", + "error_code": "", + "at": "23:24:53.381" + }, + { + "attempt": "1", + "event": "retry_scheduled", + "error_code": "injected_transient", + "at": "23:24:53.390" + }, + { + "attempt": "2", + "event": "started", + "error_code": "", + "at": "23:24:54.330" + }, + { + "attempt": "2", + "event": "retry_scheduled", + "error_code": "injected_transient", + "at": "23:24:54.339" + }, + { + "attempt": "3", + "event": "started", + "error_code": "", + "at": "23:24:56.146" + }, + { + "attempt": "3", + "event": "ready", + "error_code": "", + "at": "23:24:56.565" + } + ], + "retry_gaps_seconds": [ + 0.94, + 1.81 + ], + "expected_gap_ranges": [ + "attempt1: [0.5, 1.0] s", + "attempt2: [1.0, 2.0] s" + ], + "job_after": { + "job_id": "09973bd1-2b97-43c2-9292-d3519960a2c7", + "status": "ready", + "attempt": 3, + "max_attempts": 5, + "error_code": "", + "variant_rows": 3, + "outbox_rows": 3, + "outbox_unpublished": 0, + "staging_removed": true + }, + "ready_check": { + "variants": { + "thumbnail": "image/jpeg 256x160 5620B", + "crop": "image/jpeg 128x128 2440B", + "optimized": "image/webp 320x200 2462B" + }, + "thumbnail_real_jpeg_size": [ + 256, + 160 + ], + "original_sha256_ok": true + } +} diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..5c90482b --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/track1/backpressure.json b/evidence/track1/backpressure.json new file mode 100644 index 00000000..ef35cad3 --- /dev/null +++ b/evidence/track1/backpressure.json @@ -0,0 +1,106 @@ +{ + "experiment": "backpressure", + "started_at": "2026-11-12T07:56:12Z", + "worker_limits": "cpus=2.0 memory=768M GOMEMLIMIT=600MiB decode budget 384MiB", + "host": "Windows 11, Docker Desktop 29.7.2 (WSL2)", + "max_active_jobs_bound": 20, + "worker_concurrency": 1, + "uploads": 80, + "accepted_202": 20, + "rejected_503": 60, + "retry_after_header": [ + "5" + ], + "other": [], + "upload_burst_s": 1.22, + "drained": true, + "drain_s": 36.4, + "max_active_observed": 18, + "jobs": { + "jobs": 21, + "ready": 21, + "failed": 0, + "retried_jobs": 0, + "throughput_jobs_per_s": 0.58, + "pipeline_span_s": 36.38, + "time_to_ready_s": { + "p50": 17.728, + "p95": 32.937, + "max": 34.657 + } + }, + "after_overload_upload_status": "ready", + "after_overload_seconds_to_ready": 0.28, + "lag_timeline": [ + { + "t": 3.8, + "active": 8, + "mq_ready": 0, + "mq_unacked": 0, + "oldest_age_s": 0.29 + }, + { + "t": 7.7, + "active": 18, + "mq_ready": 18, + "mq_unacked": 1, + "oldest_age_s": 4.38 + }, + { + "t": 12.1, + "active": 16, + "mq_ready": 15, + "mq_unacked": 1, + "oldest_age_s": 8.2 + }, + { + "t": 16.0, + "active": 14, + "mq_ready": 12, + "mq_unacked": 1, + "oldest_age_s": 12.58 + }, + { + "t": 20.2, + "active": 11, + "mq_ready": 9, + "mq_unacked": 1, + "oldest_age_s": 16.32 + }, + { + "t": 23.8, + "active": 9, + "mq_ready": 9, + "mq_unacked": 1, + "oldest_age_s": 20.4 + }, + { + "t": 28.3, + "active": 7, + "mq_ready": 6, + "mq_unacked": 1, + "oldest_age_s": 24.09 + }, + { + "t": 31.5, + "active": 4, + "mq_ready": 3, + "mq_unacked": 1, + "oldest_age_s": 28.48 + }, + { + "t": 36.5, + "active": 2, + "mq_ready": 0, + "mq_unacked": 1, + "oldest_age_s": 31.68 + }, + { + "t": 40.0, + "active": 0, + "mq_ready": 0, + "mq_unacked": 0, + "oldest_age_s": 0.0 + } + ] +} diff --git a/evidence/track1/matrix.json b/evidence/track1/matrix.json new file mode 100644 index 00000000..734db2d2 --- /dev/null +++ b/evidence/track1/matrix.json @@ -0,0 +1,682 @@ +{ + "experiment": "matrix", + "started_at": "2026-11-11T23:38:36Z", + "worker_limits": "cpus=2.0 memory=768M GOMEMLIMIT=600MiB decode budget 384MiB", + "host": "Windows 11, Docker Desktop 29.7.2 (WSL2)", + "runs": [ + { + "size": "S_320x200", + "image_bytes": 111817, + "worker_concurrency_prefetch": 1, + "uploads": 120, + "accepted_202": 120, + "rejected": 0, + "upload_burst_s": 1.09, + "drained": true, + "jobs": 120, + "ready": 120, + "failed": 0, + "retried_jobs": 0, + "throughput_jobs_per_s": 9.38, + "pipeline_span_s": 12.79, + "time_to_ready_s": { + "p50": 6.325, + "p95": 11.297, + "max": 11.759 + }, + "queue_lag": { + "max_active_jobs": 51, + "max_mq_ready": 112, + "max_oldest_job_age_s": 10.94 + }, + "worker_peak_rss_mib": 59.3, + "lag_timeline": [ + { + "t": 6.9, + "active": 50, + "mq_ready": 112, + "mq_unacked": 1, + "oldest_age_s": 0.51 + }, + { + "t": 11.0, + "active": 51, + "mq_ready": 64, + "mq_unacked": 1, + "oldest_age_s": 7.09 + }, + { + "t": 16.8, + "active": 10, + "mq_ready": 12, + "mq_unacked": 1, + "oldest_age_s": 10.94 + } + ] + }, + { + "size": "M_2000x1500", + "image_bytes": 1165217, + "worker_concurrency_prefetch": 1, + "uploads": 40, + "accepted_202": 40, + "rejected": 0, + "upload_burst_s": 1.23, + "drained": true, + "jobs": 40, + "ready": 40, + "failed": 0, + "retried_jobs": 0, + "throughput_jobs_per_s": 0.56, + "pipeline_span_s": 70.84, + "time_to_ready_s": { + "p50": 36.797, + "p95": 68.11, + "max": 69.761 + }, + "queue_lag": { + "max_active_jobs": 36, + "max_mq_ready": 38, + "max_oldest_job_age_s": 69.31 + }, + "worker_peak_rss_mib": 165.3, + "lag_timeline": [ + { + "t": 6.6, + "active": 14, + "mq_ready": 38, + "mq_unacked": 1, + "oldest_age_s": 0.39 + }, + { + "t": 10.5, + "active": 36, + "mq_ready": 36, + "mq_unacked": 1, + "oldest_age_s": 7.22 + }, + { + "t": 16.4, + "active": 34, + "mq_ready": 33, + "mq_unacked": 1, + "oldest_age_s": 11.15 + }, + { + "t": 20.7, + "active": 31, + "mq_ready": 30, + "mq_unacked": 1, + "oldest_age_s": 16.91 + }, + { + "t": 25.2, + "active": 28, + "mq_ready": 27, + "mq_unacked": 1, + "oldest_age_s": 21.14 + }, + { + "t": 30.4, + "active": 26, + "mq_ready": 24, + "mq_unacked": 1, + "oldest_age_s": 25.53 + }, + { + "t": 34.5, + "active": 23, + "mq_ready": 21, + "mq_unacked": 1, + "oldest_age_s": 30.62 + }, + { + "t": 40.8, + "active": 21, + "mq_ready": 19, + "mq_unacked": 1, + "oldest_age_s": 34.71 + }, + { + "t": 44.9, + "active": 17, + "mq_ready": 16, + "mq_unacked": 1, + "oldest_age_s": 40.9 + }, + { + "t": 51.7, + "active": 15, + "mq_ready": 13, + "mq_unacked": 1, + "oldest_age_s": 44.93 + }, + { + "t": 55.8, + "active": 11, + "mq_ready": 10, + "mq_unacked": 1, + "oldest_age_s": 51.73 + }, + { + "t": 61.0, + "active": 9, + "mq_ready": 7, + "mq_unacked": 1, + "oldest_age_s": 55.64 + }, + { + "t": 65.5, + "active": 6, + "mq_ready": 5, + "mq_unacked": 1, + "oldest_age_s": 60.82 + }, + { + "t": 69.7, + "active": 3, + "mq_ready": 2, + "mq_unacked": 1, + "oldest_age_s": 65.23 + }, + { + "t": 75.2, + "active": 1, + "mq_ready": 0, + "mq_unacked": 0, + "oldest_age_s": 69.31 + } + ] + }, + { + "size": "L_5000x4000", + "image_bytes": 66065, + "worker_concurrency_prefetch": 1, + "uploads": 16, + "accepted_202": 16, + "rejected": 0, + "upload_burst_s": 0.18, + "drained": true, + "jobs": 16, + "ready": 16, + "failed": 0, + "retried_jobs": 0, + "throughput_jobs_per_s": 0.37, + "pipeline_span_s": 42.92, + "time_to_ready_s": { + "p50": 24.227, + "p95": 42.775, + "max": 42.775 + }, + "queue_lag": { + "max_active_jobs": 16, + "max_mq_ready": 14, + "max_oldest_job_age_s": 38.34 + }, + "worker_peak_rss_mib": 601.1, + "lag_timeline": [ + { + "t": 3.4, + "active": 16, + "mq_ready": 0, + "mq_unacked": 0, + "oldest_age_s": 0.4 + }, + { + "t": 8.4, + "active": 15, + "mq_ready": 14, + "mq_unacked": 1, + "oldest_age_s": 4.38 + }, + { + "t": 13.3, + "active": 13, + "mq_ready": 12, + "mq_unacked": 1, + "oldest_age_s": 9.25 + }, + { + "t": 17.9, + "active": 11, + "mq_ready": 10, + "mq_unacked": 1, + "oldest_age_s": 14.16 + }, + { + "t": 23.4, + "active": 10, + "mq_ready": 8, + "mq_unacked": 1, + "oldest_age_s": 18.7 + }, + { + "t": 27.3, + "active": 8, + "mq_ready": 6, + "mq_unacked": 1, + "oldest_age_s": 24.23 + }, + { + "t": 33.6, + "active": 6, + "mq_ready": 5, + "mq_unacked": 1, + "oldest_age_s": 28.08 + }, + { + "t": 37.6, + "active": 4, + "mq_ready": 3, + "mq_unacked": 1, + "oldest_age_s": 34.53 + }, + { + "t": 44.6, + "active": 2, + "mq_ready": 1, + "mq_unacked": 1, + "oldest_age_s": 38.34 + }, + { + "t": 48.5, + "active": 0, + "mq_ready": 0, + "mq_unacked": 0, + "oldest_age_s": 0.0 + } + ] + }, + { + "size": "S_320x200", + "image_bytes": 111817, + "worker_concurrency_prefetch": 2, + "uploads": 120, + "accepted_202": 120, + "rejected": 0, + "upload_burst_s": 1.05, + "drained": true, + "jobs": 120, + "ready": 120, + "failed": 0, + "retried_jobs": 0, + "throughput_jobs_per_s": 16.25, + "pipeline_span_s": 7.39, + "time_to_ready_s": { + "p50": 3.649, + "p95": 6.158, + "max": 6.383 + }, + "queue_lag": { + "max_active_jobs": 63, + "max_mq_ready": 64, + "max_oldest_job_age_s": 6.35 + }, + "worker_peak_rss_mib": 22.8, + "lag_timeline": [ + { + "t": 6.5, + "active": 63, + "mq_ready": 64, + "mq_unacked": 2, + "oldest_age_s": 0.51 + }, + { + "t": 10.6, + "active": 1, + "mq_ready": 0, + "mq_unacked": 0, + "oldest_age_s": 6.35 + } + ] + }, + { + "size": "M_2000x1500", + "image_bytes": 1165247, + "worker_concurrency_prefetch": 2, + "uploads": 40, + "accepted_202": 40, + "rejected": 0, + "upload_burst_s": 1.18, + "drained": true, + "jobs": 40, + "ready": 40, + "failed": 0, + "retried_jobs": 0, + "throughput_jobs_per_s": 1.11, + "pipeline_span_s": 36.02, + "time_to_ready_s": { + "p50": 19.159, + "p95": 35.018, + "max": 35.103 + }, + "queue_lag": { + "max_active_jobs": 36, + "max_mq_ready": 34, + "max_oldest_job_age_s": 33.77 + }, + "worker_peak_rss_mib": 328.2, + "lag_timeline": [ + { + "t": 3.6, + "active": 11, + "mq_ready": 0, + "mq_unacked": 0, + "oldest_age_s": 0.27 + }, + { + "t": 8.7, + "active": 36, + "mq_ready": 34, + "mq_unacked": 2, + "oldest_age_s": 4.26 + }, + { + "t": 13.5, + "active": 30, + "mq_ready": 28, + "mq_unacked": 2, + "oldest_age_s": 9.11 + }, + { + "t": 17.5, + "active": 26, + "mq_ready": 23, + "mq_unacked": 2, + "oldest_age_s": 13.82 + }, + { + "t": 23.2, + "active": 20, + "mq_ready": 17, + "mq_unacked": 2, + "oldest_age_s": 17.73 + }, + { + "t": 27.3, + "active": 14, + "mq_ready": 12, + "mq_unacked": 2, + "oldest_age_s": 23.2 + }, + { + "t": 33.9, + "active": 10, + "mq_ready": 6, + "mq_unacked": 2, + "oldest_age_s": 27.16 + }, + { + "t": 37.9, + "active": 2, + "mq_ready": 0, + "mq_unacked": 2, + "oldest_age_s": 33.77 + }, + { + "t": 44.8, + "active": 0, + "mq_ready": 0, + "mq_unacked": 0, + "oldest_age_s": 0.0 + } + ] + }, + { + "size": "L_5000x4000", + "image_bytes": 66065, + "worker_concurrency_prefetch": 2, + "uploads": 16, + "accepted_202": 16, + "rejected": 0, + "upload_burst_s": 0.18, + "drained": true, + "jobs": 16, + "ready": 16, + "failed": 0, + "retried_jobs": 0, + "throughput_jobs_per_s": 0.72, + "pipeline_span_s": 22.17, + "time_to_ready_s": { + "p50": 13.631, + "p95": 22.04, + "max": 22.04 + }, + "queue_lag": { + "max_active_jobs": 16, + "max_mq_ready": 14, + "max_oldest_job_age_s": 21.51 + }, + "worker_peak_rss_mib": 585.2, + "lag_timeline": [ + { + "t": 6.3, + "active": 16, + "mq_ready": 14, + "mq_unacked": 2, + "oldest_age_s": 0.41 + }, + { + "t": 10.4, + "active": 12, + "mq_ready": 10, + "mq_unacked": 2, + "oldest_age_s": 7.17 + }, + { + "t": 16.7, + "active": 8, + "mq_ready": 6, + "mq_unacked": 2, + "oldest_age_s": 11.19 + }, + { + "t": 20.7, + "active": 4, + "mq_ready": 2, + "mq_unacked": 2, + "oldest_age_s": 17.55 + }, + { + "t": 25.8, + "active": 2, + "mq_ready": 0, + "mq_unacked": 0, + "oldest_age_s": 21.51 + } + ] + }, + { + "size": "S_320x200", + "image_bytes": 111817, + "worker_concurrency_prefetch": 4, + "uploads": 120, + "accepted_202": 120, + "rejected": 0, + "upload_burst_s": 1.11, + "drained": true, + "jobs": 120, + "ready": 120, + "failed": 0, + "retried_jobs": 0, + "throughput_jobs_per_s": 23.27, + "pipeline_span_s": 5.16, + "time_to_ready_s": { + "p50": 2.281, + "p95": 3.945, + "max": 4.099 + }, + "queue_lag": { + "max_active_jobs": 61, + "max_mq_ready": 32, + "max_oldest_job_age_s": 0.51 + }, + "worker_peak_rss_mib": 29.0, + "lag_timeline": [ + { + "t": 4.9, + "active": 61, + "mq_ready": 32, + "mq_unacked": 4, + "oldest_age_s": 0.51 + }, + { + "t": 9.0, + "active": 0, + "mq_ready": 32, + "mq_unacked": 4, + "oldest_age_s": 0.0 + } + ] + }, + { + "size": "M_2000x1500", + "image_bytes": 1165242, + "worker_concurrency_prefetch": 4, + "uploads": 40, + "accepted_202": 40, + "rejected": 0, + "upload_burst_s": 1.28, + "drained": true, + "jobs": 40, + "ready": 40, + "failed": 0, + "retried_jobs": 0, + "throughput_jobs_per_s": 1.09, + "pipeline_span_s": 36.86, + "time_to_ready_s": { + "p50": 21.093, + "p95": 35.792, + "max": 35.83 + }, + "queue_lag": { + "max_active_jobs": 36, + "max_mq_ready": 36, + "max_oldest_job_age_s": 32.06 + }, + "worker_peak_rss_mib": 513.1, + "lag_timeline": [ + { + "t": 5.5, + "active": 9, + "mq_ready": 36, + "mq_unacked": 4, + "oldest_age_s": 0.28 + }, + { + "t": 9.4, + "active": 36, + "mq_ready": 31, + "mq_unacked": 4, + "oldest_age_s": 6.15 + }, + { + "t": 13.9, + "active": 32, + "mq_ready": 24, + "mq_unacked": 4, + "oldest_age_s": 9.87 + }, + { + "t": 19.0, + "active": 27, + "mq_ready": 20, + "mq_unacked": 4, + "oldest_age_s": 14.19 + }, + { + "t": 22.8, + "active": 20, + "mq_ready": 20, + "mq_unacked": 4, + "oldest_age_s": 19.04 + }, + { + "t": 28.5, + "active": 16, + "mq_ready": 8, + "mq_unacked": 4, + "oldest_age_s": 22.81 + }, + { + "t": 32.3, + "active": 12, + "mq_ready": 8, + "mq_unacked": 4, + "oldest_age_s": 28.43 + }, + { + "t": 39.0, + "active": 6, + "mq_ready": 0, + "mq_unacked": 0, + "oldest_age_s": 32.06 + } + ] + }, + { + "size": "L_5000x4000", + "image_bytes": 66065, + "worker_concurrency_prefetch": 4, + "uploads": 16, + "accepted_202": 16, + "rejected": 0, + "upload_burst_s": 0.18, + "drained": true, + "jobs": 16, + "ready": 16, + "failed": 0, + "retried_jobs": 0, + "throughput_jobs_per_s": 0.73, + "pipeline_span_s": 21.84, + "time_to_ready_s": { + "p50": 13.692, + "p95": 21.692, + "max": 21.692 + }, + "queue_lag": { + "max_active_jobs": 16, + "max_mq_ready": 12, + "max_oldest_job_age_s": 19.61 + }, + "worker_peak_rss_mib": 593.2, + "lag_timeline": [ + { + "t": 4.9, + "active": 16, + "mq_ready": 12, + "mq_unacked": 4, + "oldest_age_s": 0.42 + }, + { + "t": 8.9, + "active": 12, + "mq_ready": 8, + "mq_unacked": 4, + "oldest_age_s": 5.73 + }, + { + "t": 15.0, + "active": 10, + "mq_ready": 4, + "mq_unacked": 4, + "oldest_age_s": 9.71 + }, + { + "t": 18.9, + "active": 6, + "mq_ready": 2, + "mq_unacked": 4, + "oldest_age_s": 15.7 + }, + { + "t": 25.9, + "active": 2, + "mq_ready": 0, + "mq_unacked": 2, + "oldest_age_s": 19.61 + } + ] + } + ] +} diff --git a/evidence/worker-down.json b/evidence/worker-down.json new file mode 100644 index 00000000..5193415e --- /dev/null +++ b/evidence/worker-down.json @@ -0,0 +1,73 @@ +{ + "scenario": "worker-down", + "result": "PASS", + "started_at": "2026-11-11T23:23:18Z", + "attachment_id": "4ebdeb92-a3d0-467d-b8cf-e7e7d45306c2", + "message_id": "f0e4acb4-49b4-4c27-b726-911937e01552", + "upload_status": 202, + "upload_ms_worker_down": 99, + "queue_before_restart": { + "ready": 1, + "unacked": 0, + "consumers": 0 + }, + "job_before_restart": { + "job_id": "a56aa984-d3eb-4716-a47d-f246d752426c", + "status": "queued", + "attempt": 1, + "max_attempts": 5, + "error_code": "", + "variant_rows": 0, + "outbox_rows": 1, + "outbox_unpublished": 0, + "staging_removed": false + }, + "queue_after_api_and_broker_restart": { + "ready": 1, + "unacked": 0, + "consumers": 0 + }, + "status_after_restart_before_worker": "queued", + "seconds_to_ready_after_worker_start": 0.34, + "statuses_seen": [ + "processing", + "ready" + ], + "steps": [ + "docker compose stop worker", + "upload -> 202 queued", + "message with queued attachment -> 201", + "content while queued -> 409", + "plain text message -> 201", + "docker compose restart api rabbitmq (volumes kept)", + "docker compose start worker" + ], + "ready_check": { + "variants": { + "thumbnail": "image/jpeg 256x160 5620B", + "crop": "image/jpeg 128x128 2440B", + "optimized": "image/webp 320x200 2462B" + }, + "thumbnail_real_jpeg_size": [ + 256, + 160 + ], + "original_sha256_ok": true + }, + "job_after": { + "job_id": "a56aa984-d3eb-4716-a47d-f246d752426c", + "status": "ready", + "attempt": 1, + "max_attempts": 5, + "error_code": "", + "variant_rows": 3, + "outbox_rows": 1, + "outbox_unpublished": 0, + "staging_removed": true + }, + "s3": { + "variants_objects": 3, + "final_original": 1, + "staging_objects": 0 + } +} diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..e8c33608 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,45 @@ +global + log stdout format raw local0 info + maxconn 4096 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold obsolete 5s + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlognull + timeout connect 1s + timeout client 75s + timeout server 75s + timeout http-request 5s + timeout http-keep-alive 10s + timeout check 500ms + retries 2 + retry-on conn-failure + option redispatch 1 + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 init-addr none resolvers docker resolve-prefer ipv4 + server-template api 8 api:8080 + +frontend internal_stats + bind :8404 + monitor-uri /healthz + stats enable + stats uri /stats + stats refresh 5s diff --git a/infra/valkey/valkey.conf b/infra/valkey/valkey.conf new file mode 100644 index 00000000..6e50813e --- /dev/null +++ b/infra/valkey/valkey.conf @@ -0,0 +1,8 @@ +bind 0.0.0.0 +port 6379 +protected-mode yes +save "" +appendonly no +maxmemory 64mb +maxmemory-policy volatile-ttl +include /run/secrets/valkey_auth diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 00000000..5bb9586d --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +import json +import secrets +import shutil +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRETS = ROOT / 'secrets' + + +def write(name, value): + path = SECRETS / name + if path.exists(): + print('keep secrets/' + name) + return path.read_text().strip() + path.write_bytes((value + '\n').encode('utf-8')) + print('created secrets/' + name) + return value + + +def main(): + SECRETS.mkdir(exist_ok=True) + pg = write('pg_password', secrets.token_urlsafe(24)) + write('database_url', 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' % pg) + vk_conf = SECRETS / 'valkey_auth.conf' + if vk_conf.exists(): + vk = vk_conf.read_text().split()[-1] + print('keep secrets/valkey_auth.conf') + else: + vk = secrets.token_urlsafe(24) + write('valkey_auth.conf', 'requirepass ' + vk) + write('cache_url', 'redis://:%s@valkey:6379/0' % vk) + admin_id = write('s3_admin_access_key_id', 'admin' + secrets.token_hex(8)) + admin_secret = write('s3_admin_secret_access_key', secrets.token_urlsafe(30)) + api_id = write('s3_access_key_id', 'api' + secrets.token_hex(8)) + api_secret = write('s3_secret_access_key', secrets.token_urlsafe(30)) + bucket = 'messenger-attachments' + identities = {'identities': [ + {'name': 'bootstrap-admin', + 'credentials': [{'accessKey': admin_id, 'secretKey': admin_secret}], + 'actions': ['Admin', 'Read', 'Write', 'List', 'Tagging']}, + {'name': 'messenger-api', + 'credentials': [{'accessKey': api_id, 'secretKey': api_secret}], + 'actions': ['Read:' + bucket, 'Write:' + bucket, 'List:' + bucket]}, + ]} + write('s3_identities.json', json.dumps(identities, indent=2)) + mq_conf = SECRETS / 'rabbitmq_auth.conf' + if mq_conf.exists(): + mq = [l.split('=', 1)[1].strip() for l in mq_conf.read_text().splitlines() if l.startswith('default_pass')][0] + print('keep secrets/rabbitmq_auth.conf') + else: + mq = secrets.token_urlsafe(24) + write('rabbitmq_auth.conf', 'default_user = messenger\ndefault_pass = %s\ndefault_vhost = /' % mq) + write('broker_url', 'amqp://messenger:%s@rabbitmq:5672/' % mq) + env = ROOT / '.env' + if not env.exists(): + shutil.copy(ROOT / '.env.example', env) + print('created .env from .env.example') + + +if __name__ == '__main__': + main() diff --git a/scripts/lab5ops.py b/scripts/lab5ops.py new file mode 100644 index 00000000..6736469a --- /dev/null +++ b/scripts/lab5ops.py @@ -0,0 +1,428 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import threading +import time +import zlib + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +sys.path.insert(0, str(ROOT / 'student-tests')) +sys.path.insert(0, str(ROOT / 'scripts')) +from client import Client, Failure, check +from image_probe import jpeg_size +import imagegen +from lablib import sh + +PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes() +NETWORK = 'mtusi-lab5_data' +TOOL_IMAGE = 'python:3.12.8-alpine3.21' + + +def s3tool(*args): + cmd = ['docker', 'run', '--rm', '--network', NETWORK, + '-v', str(ROOT / 'secrets').replace('\\', '/') + ':/secrets:ro', + '-v', str(ROOT / 'scripts').replace('\\', '/') + ':/tool:ro', + TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', *args] + r = subprocess.run(cmd, capture_output=True, text=True, encoding='utf-8') + if r.returncode != 0: + raise RuntimeError('s3tool failed: ' + (r.stdout + r.stderr).strip()[-300:]) + return json.loads(r.stdout.strip().splitlines()[-1]) + + +def psql(sql): + return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', + '-At', '-F', '|', '-v', 'ON_ERROR_STOP=1', '-c', sql) + + +def queues(): + out = sh('docker', 'compose', 'exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_queues', + 'name', 'messages_ready', 'messages_unacknowledged', 'consumers', '--formatter', 'json') + return {q['name']: {'ready': q['messages_ready'], 'unacked': q['messages_unacknowledged'], + 'consumers': q['consumers']} for q in json.loads(out)} + + +def restarts(service): + out = sh('docker', 'compose', 'ps', '-a', '-q', service) + return {cid[:12]: int(sh('docker', 'inspect', '-f', '{{.RestartCount}}', cid)) for cid in out.split()} + + +def events(att_id): + rows = psql("SELECT e.attempt, e.event, COALESCE(e.error_code,''), " + "to_char(e.created_at AT TIME ZONE 'UTC','HH24:MI:SS.MS') " + "FROM job_events e JOIN attachment_jobs j USING (job_id) " + "WHERE j.attachment_id = '%s' ORDER BY e.id" % att_id) + return [dict(zip(('attempt', 'event', 'error_code', 'at'), r.split('|'))) for r in rows.splitlines()] + + +def event_times(att_id): + rows = psql("SELECT e.event, extract(epoch FROM e.created_at) FROM job_events e JOIN attachment_jobs j USING (job_id) " + "WHERE j.attachment_id = '%s' ORDER BY e.id" % att_id) + return [(r.split('|')[0], float(r.split('|')[1])) for r in rows.splitlines()] + + +def job_row(att_id): + r = psql("SELECT j.job_id, j.status, j.attempt, j.max_attempts, COALESCE(j.error_code,''), " + "(SELECT count(*) FROM attachment_variants v WHERE v.attachment_id = j.attachment_id), " + "(SELECT count(*) FROM job_outbox o WHERE o.job_id = j.job_id), " + "(SELECT count(*) FROM job_outbox o WHERE o.job_id = j.job_id AND o.published_at IS NULL), " + "a.staging_key IS NULL " + "FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE j.attachment_id = '%s'" % att_id) + k = ('job_id', 'status', 'attempt', 'max_attempts', 'error_code', 'variant_rows', 'outbox_rows', + 'outbox_unpublished', 'staging_removed') + v = r.split('|') + d = dict(zip(k, v)) + for f in ('attempt', 'max_attempts', 'variant_rows', 'outbox_rows', 'outbox_unpublished'): + d[f] = int(d[f]) + d['staging_removed'] = d['staging_removed'] == 't' + return d + + +def s3_counts(chat_id, att_id): + return {'variants_objects': s3tool('count', 'variants/%s/%s/' % (chat_id, att_id))['count'], + 'final_original': s3tool('count', 'attachments/%s/%s' % (chat_id, att_id))['count'], + 'staging_objects': s3tool('count', 'staging/' + att_id)['count']} + + +def upload(c, actor, chat, data=PNG, name='sample.png', ctype='image/png', expected=202): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + name, expected, + actor=actor, raw=data, headers={'Content-Type': ctype}, + schema='Attachment' if expected == 202 else None) + + +def wait_status(c, actor, att_id, want, timeout=120): + deadline = time.monotonic() + timeout + seen = [] + while time.monotonic() < deadline: + try: + a = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json'] + except Failure: + time.sleep(0.5) + continue + if not seen or seen[-1] != a['status']: + seen.append(a['status']) + if a['status'] in want: + return a, seen + time.sleep(0.3) + raise Failure('attachment did not reach %s in %ss (seen %s)' % (want, timeout, seen)) + + +def wait_api(c, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + c.call('GET', '/health/ready', schema='Health') + return + except Failure: + time.sleep(0.5) + raise Failure('API not ready') + + +def check_ready(c, actor, att): + names = sorted(v['name'] for v in att['variants']) + check(names == ['crop', 'optimized', 'thumbnail'], 'variants: %s' % names) + orig = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=actor) + check(hashlib.sha256(orig['bytes']).hexdigest() == att['sha256'], 'original changed') + out = {'variants': {}} + for v in att['variants']: + r = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + v['name'], actor=actor) + check(hashlib.sha256(r['bytes']).hexdigest() == v['sha256'], v['name'] + ' checksum') + check(r['headers'].get_content_type() == v['content_type'], v['name'] + ' content type') + out['variants'][v['name']] = '%s %dx%d %dB' % (v['content_type'], v['width'], v['height'], v['size_bytes']) + thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=actor)['bytes'] + out['thumbnail_real_jpeg_size'] = list(jpeg_size(thumb)) + out['original_sha256_ok'] = True + return out + + +def fault_upload(c, actor, chat, point): + name = 'fault-%s-%d.png' % (point, int(time.time() * 1000)) + try: + att = upload(c, actor, chat, name=name)['json'] + return att['id'], name, 202 + except Failure as exc: + att_id = psql("SELECT id FROM attachments WHERE chat_id = '%s' AND filename = '%s'" % (chat['id'], name)) + return att_id, name, 'client saw failure: ' + str(exc)[-60:] + + +def cmd_worker_down(a): + c = Client(a.base_url, timeout=30) + alice, bob, _eve, chat = c.bootstrap() + steps = [] + sh('docker', 'compose', 'stop', 'worker') + steps.append('docker compose stop worker') + t0 = time.monotonic() + r = upload(c, alice, chat) + att = r['json'] + upload_ms = round((time.monotonic() - t0) * 1000) + steps.append('upload -> %d %s' % (r['status'], att['status'])) + msg = c.message(alice, chat, '', attachments=[att['id']])['json'] + steps.append('message with queued attachment -> 201') + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob) + steps.append('content while queued -> 409') + c.message(bob, chat, 'текст работает без worker') + steps.append('plain text message -> 201') + time.sleep(2) + q_before = queues() + job_before = job_row(att['id']) + sh('docker', 'compose', 'restart', 'api') + sh('docker', 'compose', 'restart', 'rabbitmq') + sh('docker', 'compose', 'up', '-d', '--no-recreate', '--wait', 'rabbitmq', 'api', 'lb') + wait_api(c) + steps.append('docker compose restart api rabbitmq (volumes kept)') + q_after = queues() + still = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status'] + sh('docker', 'compose', 'start', 'worker') + steps.append('docker compose start worker') + t1 = time.monotonic() + ready, seen = wait_status(c, bob, att['id'], ('ready', 'failed')) + check(ready['status'] == 'ready', 'did not become ready') + res = { + 'attachment_id': att['id'], 'message_id': msg['id'], + 'upload_status': r['status'], 'upload_ms_worker_down': upload_ms, + 'queue_before_restart': q_before.get('attachment.process.v1'), + 'job_before_restart': job_before, + 'queue_after_api_and_broker_restart': q_after.get('attachment.process.v1'), + 'status_after_restart_before_worker': still, + 'seconds_to_ready_after_worker_start': round(time.monotonic() - t1, 2), + 'statuses_seen': seen, 'steps': steps, + 'ready_check': check_ready(c, bob, ready), + 'job_after': job_row(att['id']), 's3': s3_counts(chat['id'], att['id']), + } + return res + + +def cmd_api_crash(a): + c = Client(a.base_url, timeout=30) + alice, bob, _eve, chat = c.bootstrap() + before = restarts('api') + att_id, name, client_view = fault_upload(c, alice, chat, 'api-after-commit') + check(att_id, 'attachment row not found') + row = job_row(att_id) + ready, seen = wait_status(c, bob, att_id, ('ready', 'failed')) + check(ready['status'] == 'ready', 'not ready') + time.sleep(3) + return {'fault': 'API os.Exit right after COMMIT(queued+job+outbox), before publish and before HTTP response', + 'attachment_id': att_id, 'client_view': client_view, + 'job_right_after_crash': row, 'api_restart_counts_before': before, 'api_restart_counts_after': restarts('api'), + 'statuses_seen': seen, 'events': events(att_id), 'job_after': job_row(att_id), + 'ready_check': check_ready(c, bob, ready)} + + +def _worker_crash(a, point, desc): + c = Client(a.base_url, timeout=30) + alice, bob, _eve, chat = c.bootstrap() + before = restarts('worker') + att_id, name, client_view = fault_upload(c, alice, chat, point) + ready, seen = wait_status(c, bob, att_id, ('ready', 'failed')) + check(ready['status'] == 'ready', 'not ready') + time.sleep(5) + job = job_row(att_id) + s3 = s3_counts(chat['id'], att_id) + check(job['variant_rows'] == 3 and s3['variants_objects'] == 3, 'duplicated or missing variants') + logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '3m') + noop = [json.loads(l) for l in logs.splitlines() if att_id in l and 'no-op' in l] + return {'fault': desc, 'attachment_id': att_id, 'upload': client_view, + 'worker_restart_counts_before': before, 'worker_restart_counts_after': restarts('worker'), + 'statuses_seen': seen, 'events': events(att_id), 'job_after': job, 's3': s3, + 'redelivery_noop_log': [{'result': n.get('result'), 'redelivered': n.get('redelivered')} for n in noop], + 'ready_check': check_ready(c, bob, ready)} + + +def cmd_effect_crash(a): + return _worker_crash(a, 'worker-after-effect', + 'worker os.Exit after writing original+variants to S3, before result COMMIT and ack') + + +def cmd_ack_crash(a): + return _worker_crash(a, 'worker-before-ack', 'worker os.Exit after result COMMIT + staging cleanup, before ack') + + +def cmd_retry(a): + c = Client(a.base_url, timeout=30) + alice, bob, _eve, chat = c.bootstrap() + att_id, _, _ = fault_upload(c, alice, chat, 'transient') + ready, seen = wait_status(c, bob, att_id, ('ready', 'failed')) + check(ready['status'] == 'ready', 'not ready') + t = event_times(att_id) + fails = [x for x in t if x[0] == 'retry_scheduled'] + starts = [x for x in t if x[0] == 'started'] + gaps = [round(starts[i + 1][1] - fails[i][1], 2) for i in range(len(fails))] + return {'fault': 'attempts 1 and 2 raise a transient error (injected_transient)', + 'attachment_id': att_id, 'statuses_seen': seen, 'events': events(att_id), + 'retry_gaps_seconds': gaps, + 'expected_gap_ranges': ['attempt1: backoff [0.5, 1.0] s + relay poll <= 1 s', + 'attempt2: backoff [1.0, 2.0] s + relay poll <= 1 s'], + 'job_after': job_row(att_id), 'ready_check': check_ready(c, bob, ready)} + + +def cmd_replay(a): + c = Client(a.base_url, timeout=30) + alice, bob, _eve, chat = c.bootstrap() + att_id, _, _ = fault_upload(c, alice, chat, 'flaky') + failed, seen = wait_status(c, bob, att_id, ('ready', 'failed'), timeout=180) + check(failed['status'] == 'failed', 'expected failed') + res = {'fault': 'transient error on every attempt <= JOB_MAX_ATTEMPTS', 'attachment_id': att_id, + 'statuses_seen': seen, 'failed_metadata': {k: failed[k] for k in ('status', 'error_code', 'variants')}, + 'content_when_failed': c.call('GET', '/api/v1/attachments/' + att_id + '/content', 409, actor=bob)['status'], + 'attach_failed_to_message': c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, + body={'text': 'x', 'attachment_ids': [att_id]}, + headers={'Idempotency-Key': 'replay-' + att_id})['status'], + 'job_failed': job_row(att_id)} + res['dlq_registry_query'] = psql("SELECT count(*) FROM attachment_jobs WHERE status = 'failed' AND attachment_id = '%s'" % att_id) + replay = lambda: json.loads(sh('docker', 'compose', 'exec', '-T', 'worker', '/app/server', 'replay', att_id, + '--actor', 'lab5ops')) + res['replay_1'] = replay() + ready, seen2 = wait_status(c, bob, att_id, ('ready', 'failed')) + check(ready['status'] == 'ready', 'replay did not finish') + res['replay_2_after_ready'] = replay() + res['statuses_after_replay'] = seen2 + res['events'] = events(att_id) + res['job_after'] = job_row(att_id) + res['ready_check'] = check_ready(c, bob, ready) + return res + + +def corrupt_png(): + good = imagegen.png(64, 64) + return (b'\x89PNG\r\n\x1a\n' + imagegen.chunk(b'IHDR', good[16:29]) + + imagegen.chunk(b'IDAT', b'not-a-zlib-stream' * 10) + imagegen.chunk(b'IEND', b'')) + + +def cmd_invalid(a): + c = Client(a.base_url, timeout=60) + alice, bob, _eve, chat = c.bootstrap() + att = upload(c, alice, chat, data=corrupt_png(), name='corrupt.png')['json'] + failed, seen = wait_status(c, bob, att['id'], ('ready', 'failed')) + check(failed['status'] == 'failed' and failed['error_code'] == 'invalid_image', 'expected invalid_image') + return { + 'corrupt_png': {'upload': 202, 'statuses_seen': seen, 'error_code': failed['error_code'], + 'events': events(att['id']), 'job': job_row(att['id'])}, + 'api_rejects_before_queue': { + 'pixel_bomb_20000x20000': upload(c, alice, chat, data=imagegen.bomb_png(), name='bomb.png', expected=413)['status'], + 'not_an_image': upload(c, alice, chat, data=b'GIF89a' + b'\0' * 100, name='x.png', expected=415)['status'], + 'over_10MiB': upload(c, alice, chat, data=b'\x89PNG\r\n\x1a\n' + b'\0' * (10 * 1024 * 1024 + 1), + name='big.png', expected=413)['status'], + 'path_in_filename': upload(c, alice, chat, name='..%2Fetc.png', expected=400)['status'], + }} + + +def cmd_hang(a): + c = Client(a.base_url, timeout=30) + alice, bob, _eve, chat = c.bootstrap() + att_id, _, _ = fault_upload(c, alice, chat, 'hang') + t0 = time.monotonic() + ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'), timeout=150) + check(ready['status'] == 'ready', 'not ready') + took = round(time.monotonic() - t0, 1) + time.sleep(70) + logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '5m') + stale = [l for l in logs.splitlines() if att_id in l and ('stale attempt' in l or 'no-op' in l)] + return {'fault': 'attempt 1 freezes for 3x lease without heartbeat (simulated hung worker)', + 'attachment_id': att_id, 'seconds_to_ready': took, 'statuses_seen': seen, + 'events': events(att_id), 'job_after': job_row(att_id), 's3': s3_counts(chat['id'], att_id), + 'stale_attempt_logs': [json.loads(l).get('msg') for l in stale], + 'ready_check': check_ready(c, bob, ready)} + + +def cmd_dlq(a): + before = queues() + script = ('U=$(sed -n "s/^default_user = //p" /etc/rabbitmq/conf.d/20-auth.conf); ' + 'P=$(sed -n "s/^default_pass = //p" /etc/rabbitmq/conf.d/20-auth.conf); ' + 'wget -q -O- --header "Content-Type: application/json" --post-data ' + '\'{"properties":{"delivery_mode":2},"routing_key":"attachment.process.v1",' + '"payload":"{\\"not\\":\\"a job\\"}","payload_encoding":"string"}\' ' + '"http://$U:$P@127.0.0.1:15672/api/exchanges/%2F/attachments/publish"') + routed = sh('docker', 'compose', 'exec', '-T', 'rabbitmq', 'sh', '-c', script) + time.sleep(3) + after = queues() + logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '1m') + return {'published': json.loads(routed), 'dlq_before': before.get('attachment.process.dlq'), + 'dlq_after': after.get('attachment.process.dlq'), + 'worker_log': [json.loads(l)['msg'] for l in logs.splitlines() if 'DLQ' in l][-1:]} + + +def gradient_png(w, h): + row = b'\x02' + bytes(3 * w) + first = b'\x00' + bytes((x * 255 // max(1, w - 1)) for x in range(w) for _ in range(3)) + raw = first + row * (h - 1) + return imagegen.png(w, h, raw=raw, level=9) + + +class MemSampler: + def __init__(self, service): + self.cids = sh('docker', 'compose', 'ps', '-q', service).split() + self.peak = 0.0 + self._stop = threading.Event() + self.t = threading.Thread(target=self.run, daemon=True) + + def run(self): + while not self._stop.is_set(): + out = sh('docker', 'stats', '--no-stream', '--format', '{{.MemUsage}}', *self.cids, check=False) + for line in out.splitlines(): + used = line.split('/')[0].strip() + num = float(''.join(ch for ch in used if ch.isdigit() or ch == '.') or 0) + mib = num * 1024 if used.endswith('GiB') else num / 1024 if used.endswith('KiB') else num + self.peak = max(self.peak, mib) + + def __enter__(self): + self.t.start() + return self + + def __exit__(self, *exc): + self._stop.set() + self.t.join(timeout=10) + + +def cmd_limits(a): + c = Client(a.base_url, timeout=60) + alice, bob, _eve, chat = c.bootstrap() + data = gradient_png(6000, 6000) + limit = sh('docker', 'inspect', '-f', '{{.HostConfig.Memory}}', sh('docker', 'compose', 'ps', '-q', 'worker').split()[0]) + with MemSampler('worker') as m: + t0 = time.monotonic() + att = upload(c, alice, chat, data=data, name='big-6000x6000.png')['json'] + ready, seen = wait_status(c, bob, att['id'], ('ready', 'failed'), timeout=120) + took = round(time.monotonic() - t0, 1) + time.sleep(2) + check(ready['status'] == 'ready', 'big image failed: %s' % ready['error_code']) + return {'image': '6000x6000 PNG (36 MPx, %d bytes)' % len(data), 'seconds_to_ready': took, + 'worker_peak_mem_mib': round(m.peak, 1), 'worker_mem_limit_mib': int(limit) // (1 << 20), + 'variants': {v['name']: '%dx%d' % (v['width'], v['height']) for v in ready['variants']}, + 'statuses_seen': seen, 'job': job_row(att['id'])} + + +COMMANDS = { + 'worker-down': cmd_worker_down, 'api-crash': cmd_api_crash, 'effect-crash': cmd_effect_crash, + 'ack-crash': cmd_ack_crash, 'retry': cmd_retry, 'replay': cmd_replay, 'invalid': cmd_invalid, + 'hang': cmd_hang, 'dlq': cmd_dlq, 'limits': cmd_limits, +} + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument('command', choices=sorted(COMMANDS)) + p.add_argument('--base-url', default='http://localhost:8080') + p.add_argument('--output') + a = p.parse_args() + started = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) + try: + res = COMMANDS[a.command](a) + res = {'scenario': a.command, 'result': 'PASS', 'started_at': started, **res} + except (Failure, RuntimeError) as exc: + res = {'scenario': a.command, 'result': 'FAIL', 'started_at': started, 'error': str(exc)} + text = json.dumps(res, ensure_ascii=False, indent=2) + print(text) + if a.output: + Path(ROOT / a.output).parent.mkdir(parents=True, exist_ok=True) + (ROOT / a.output).write_text(text + '\n', encoding='utf-8') + if res['result'] != 'PASS': + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/scripts/lablib.py b/scripts/lablib.py new file mode 100644 index 00000000..64b136c3 --- /dev/null +++ b/scripts/lablib.py @@ -0,0 +1,130 @@ +import json +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure + +PROJECT = 'mtusi-lab5' + + +def sh(*args, check=True): + r = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding='utf-8') + if check and r.returncode != 0: + raise RuntimeError('%s failed: %s' % (' '.join(args[:3]), r.stderr.strip()[:300])) + return r.stdout.strip() + + +def api_containers(all_states=False): + out = sh('docker', 'ps', *(['-a'] if all_states else []), '--no-trunc', + '--filter', 'label=com.docker.compose.project=' + PROJECT, + '--filter', 'label=com.docker.compose.service=api', + '--format', '{{json .}}') + rows = [] + for line in out.splitlines(): + d = json.loads(line) + rows.append({'id': d['ID'][:12], 'name': d['Names'], 'state': d['State'], 'status': d['Status']}) + return sorted(rows, key=lambda r: r['name']) + + +def health(cid): + return sh('docker', 'inspect', '-f', '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}', cid) + + +def wait_healthy(cid, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if health(cid) == 'healthy': + return True + time.sleep(0.25) + return False + + +def ps_snapshot(): + return sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}}\t{{.State}}\t{{.Status}}') + + +class WriteLoad: + + def __init__(self, base_url, workers=8, timeout=2.0, pause=0.0): + self.base = base_url + self.workers = workers + self.timeout = timeout + self.pause = pause + self.events = [] + self.confirmed = [] + self.retries = 0 + self._lock = threading.Lock() + self._stop = threading.Event() + self._threads = [] + c = Client(base_url, timeout=10) + self.alice, self.bob, _eve, self.chat = c.bootstrap() + self.reader = c + + def _worker(self, wid): + c = Client(self.base, timeout=self.timeout) + key = text = None + n = 0 + while not self._stop.is_set(): + if key is None: + key = uuid.uuid4().hex + text = 'load-%d-%d-%s' % (wid, n, key[:8]) + start = time.monotonic() + try: + r = c.message(self.alice, self.chat, text, key=key) + with self._lock: + self.events.append((start, True, r['headers']['X-Instance-Id'], time.monotonic() - start)) + self.confirmed.append(r['json']) + key = None + n += 1 + except Failure as exc: + with self._lock: + self.events.append((start, False, str(exc)[-60:], time.monotonic() - start)) + self.retries += 1 + time.sleep(0.05) + if self.pause: + time.sleep(self.pause) + + def start(self): + for i in range(self.workers): + t = threading.Thread(target=self._worker, args=(i,), daemon=True) + t.start() + self._threads.append(t) + self.t0 = time.monotonic() + + def stop(self): + self._stop.set() + for t in self._threads: + t.join(timeout=15) + + def window(self, a, b): + ev = [e for e in self.events if a <= e[0] < b] + ok = [e for e in ev if e[1]] + lat = sorted(e[3] for e in ok) + pct = lambda p: round(1000 * lat[min(len(lat) - 1, int(p * len(lat)))], 1) if lat else None + return {'seconds': round(b - a, 2), 'ok': len(ok), 'errors': len(ev) - len(ok), + 'ok_rps': round(len(ok) / (b - a), 1) if b > a else None, + 'p50_ms': pct(.5), 'p95_ms': pct(.95), 'p99_ms': pct(.99)} + + def verify_history(self): + items, cursor = [], None + path = '/api/v1/chats/' + self.chat['id'] + '/messages?limit=100' + while True: + page = self.reader.call('GET', path + ('&cursor=' + cursor if cursor else ''), + actor=self.bob, schema='MessagePage')['json'] + items += page['items'] + cursor = page['next_cursor'] + if not cursor: + break + by_id = {m['id']: m for m in items} + texts = [m['text'] for m in items] + lost = [m for m in self.confirmed if by_id.get(m['id']) != m] + return {'stored_messages': len(items), 'confirmed_201': len(self.confirmed), + 'lost_confirmed': len(lost), 'duplicate_ids': len(items) - len(by_id), + 'duplicate_texts': len(texts) - len(set(texts)), + 'unique_confirmed': len({m['id'] for m in self.confirmed})} diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..3338b063 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,6 @@ +@echo off +cd /d "%~dp0.." +docker compose stop lb api worker || exit /b 1 +docker compose restart postgres valkey s3 rabbitmq || exit /b 1 +docker compose up -d --no-recreate --wait postgres valkey s3 rabbitmq || exit /b 1 +docker compose start api worker lb || exit /b 1 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..12268be9 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose stop lb api worker +docker compose restart postgres valkey s3 rabbitmq +docker compose up -d --no-recreate --wait postgres valkey s3 rabbitmq +docker compose start api worker lb diff --git a/scripts/s3tool.py b/scripts/s3tool.py new file mode 100644 index 00000000..7be9505f --- /dev/null +++ b/scripts/s3tool.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +import argparse +from concurrent.futures import ThreadPoolExecutor +import datetime as dt +import hashlib +import hmac +import http.client +import json +import os +import sys +import threading +from urllib.parse import quote +import xml.etree.ElementTree as ET + +HOST = os.environ.get('S3_HOST', 's3:8333') +REGION = os.environ.get('S3_REGION', 'us-east-1') +BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments') +SECRETS = os.environ.get('SECRETS_DIR', '/secrets') +NS = '{http://s3.amazonaws.com/doc/2006-03-01/}' + + +def creds(identity): + if identity == 'anonymous': + return None + prefix = 's3_admin_' if identity == 'admin' else 's3_' + read = lambda n: open(os.path.join(SECRETS, prefix + n)).read().strip() + return read('access_key_id'), read('secret_access_key') + + +def _h(key, msg): + return hmac.new(key, msg.encode(), hashlib.sha256).digest() + + +_local = threading.local() + + +def request(method, path, identity='api', body=b'', query=None): + query = query or {} + headers = {'host': HOST} + payload_hash = hashlib.sha256(body).hexdigest() + c = creds(identity) + if c: + now = dt.datetime.now(dt.timezone.utc) + amzdate, date = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d') + headers.update({'x-amz-content-sha256': payload_hash, 'x-amz-date': amzdate}) + signed = ';'.join(sorted(headers)) + cq = '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items())) + creq = '\n'.join([method, quote(path, safe='/-_.~'), cq, + ''.join('%s:%s\n' % (k, headers[k]) for k in sorted(headers)), signed, payload_hash]) + scope = '%s/%s/s3/aws4_request' % (date, REGION) + sts = '\n'.join(['AWS4-HMAC-SHA256', amzdate, scope, hashlib.sha256(creq.encode()).hexdigest()]) + k = _h(_h(_h(_h(('AWS4' + c[1]).encode(), date), REGION), 's3'), 'aws4_request') + sig = hmac.new(k, sts.encode(), hashlib.sha256).hexdigest() + headers['authorization'] = 'AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s' % ( + c[0], scope, signed, sig) + url = quote(path, safe='/-_.~') + if query: + url += '?' + '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items())) + conn = getattr(_local, 'conn', None) + if conn is None: + conn = _local.conn = http.client.HTTPConnection(HOST, timeout=30) + try: + conn.request(method, url, body=body or None, headers=headers) + r = conn.getresponse() + return r.status, r.read() + except (OSError, http.client.HTTPException): + _local.conn = None + raise + + +def obj(key): + return '/' + BUCKET + '/' + key + + +def list_keys(prefix, identity='api'): + keys, token = [], None + while True: + q = {'list-type': '2', 'prefix': prefix, 'max-keys': '1000'} + if token: + q['continuation-token'] = token + st, body = request('GET', '/' + BUCKET, identity, query=q) + if st != 200: + raise SystemExit('list failed: HTTP %d' % st) + root = ET.fromstring(body) + keys += [e.find(NS + 'Key').text for e in root.findall(NS + 'Contents')] + if root.findtext(NS + 'IsTruncated') != 'true': + return keys + token = root.findtext(NS + 'NextContinuationToken') + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('cmd', choices=['get', 'put', 'delete', 'create-bucket', 'list', 'count', 'bulk-put', 'bulk-delete']) + p.add_argument('arg', nargs='?', default='') + p.add_argument('--as', dest='identity', default='api', choices=['api', 'admin', 'anonymous']) + p.add_argument('--size', type=int, default=100) + p.add_argument('--workers', type=int, default=32) + p.add_argument('--stdin', action='store_true', help='put: body from stdin instead of random bytes') + a = p.parse_args() + out = {'cmd': a.cmd, 'as': a.identity} + if a.cmd == 'get': + st, body = request('GET', obj(a.arg), a.identity) + out.update(status=st, bytes=len(body) if st == 200 else 0, + sha256=hashlib.sha256(body).hexdigest() if st == 200 else None) + elif a.cmd == 'put': + body = sys.stdin.buffer.read() if a.stdin else os.urandom(a.size) + st, _ = request('PUT', obj(a.arg), a.identity, body=body) + out.update(status=st) + elif a.cmd == 'delete': + st, _ = request('DELETE', obj(a.arg), a.identity) + out.update(status=st) + elif a.cmd == 'create-bucket': + st, _ = request('PUT', '/' + a.arg, a.identity) + out.update(status=st) + elif a.cmd == 'list': + out.update(keys=list_keys(a.arg, a.identity)) + elif a.cmd == 'count': + out.update(count=len(list_keys(a.arg, a.identity))) + elif a.cmd in ('bulk-put', 'bulk-delete'): + keys = [k.strip() for k in sys.stdin if k.strip()] + data = b'x' * a.size + + def one(k): + if a.cmd == 'bulk-put': + return request('PUT', obj(k), a.identity, body=data)[0] + return request('DELETE', obj(k), a.identity)[0] + + start = dt.datetime.now() + with ThreadPoolExecutor(a.workers) as pool: + codes = list(pool.map(one, keys)) + out.update(count=len(keys), ok=sum(1 for c in codes if c in (200, 204)), + seconds=round((dt.datetime.now() - start).total_seconds(), 2)) + print(json.dumps(out)) + + +if __name__ == '__main__': + main() diff --git a/scripts/start-worker.cmd b/scripts/start-worker.cmd new file mode 100644 index 00000000..86256a13 --- /dev/null +++ b/scripts/start-worker.cmd @@ -0,0 +1,3 @@ +@echo off +cd /d "%~dp0.." +docker compose start worker diff --git a/scripts/start-worker.sh b/scripts/start-worker.sh new file mode 100644 index 00000000..3f8eb195 --- /dev/null +++ b/scripts/start-worker.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose start worker diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..ba07db4e --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,10 @@ +@echo off +setlocal +cd /d "%~dp0.." +if "%TARGET_INSTANCE_ID%"=="" exit /b 2 +echo %TARGET_INSTANCE_ID%| findstr /r /x "[0-9a-f][0-9a-f]*" >nul || exit /b 2 +set CID= +for /f %%i in ('docker ps -q --filter "id=%TARGET_INSTANCE_ID%" --filter "label=com.docker.compose.project=mtusi-lab5" --filter "label=com.docker.compose.service=api"') do set CID=%%i +if "%CID%"=="" exit /b 3 +docker stop -t 15 %CID% >nul || exit /b 1 +echo stopped api container %CID% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..3c86367a --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +case "$TARGET_INSTANCE_ID" in + *[!0-9a-f]*) echo "unexpected instance id format" >&2; exit 2 ;; +esac +cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \ + --filter "label=com.docker.compose.project=mtusi-lab5" \ + --filter "label=com.docker.compose.service=api") +[ -n "$cid" ] || { echo "no running api replica matches" >&2; exit 3; } +[ "$(printf '%s\n' "$cid" | wc -l)" -eq 1 ] || { echo "ambiguous id" >&2; exit 3; } +docker stop -t 15 "$cid" >/dev/null +echo "stopped api container $cid" diff --git a/scripts/stop-worker.cmd b/scripts/stop-worker.cmd new file mode 100644 index 00000000..e24d1c97 --- /dev/null +++ b/scripts/stop-worker.cmd @@ -0,0 +1,3 @@ +@echo off +cd /d "%~dp0.." +docker compose stop worker diff --git a/scripts/stop-worker.sh b/scripts/stop-worker.sh new file mode 100644 index 00000000..8926fdc4 --- /dev/null +++ b/scripts/stop-worker.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose stop worker diff --git a/scripts/track1/run_track1.py b/scripts/track1/run_track1.py new file mode 100644 index 00000000..348e03eb --- /dev/null +++ b/scripts/track1/run_track1.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +import argparse +from concurrent.futures import ThreadPoolExecutor +import json +import os +from pathlib import Path +import statistics +import subprocess +import sys +import threading +import time + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'tests')) +sys.path.insert(0, str(ROOT / 'scripts')) +sys.path.insert(0, str(ROOT / 'student-tests')) +from client import Client, Failure +from lab5ops import MemSampler, gradient_png, psql, queues +import imagegen + + +def compose(*args, env=None): + e = dict(os.environ) + e.update(env or {}) + r = subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, env=e, + capture_output=True, text=True, encoding='utf-8') + if r.returncode != 0: + raise RuntimeError('compose %s: %s' % (args[:2], r.stderr[-300:])) + + +def noisy_png(w, h): + row_noise = lambda: b'\x00' + os.urandom(3 * w) + row_same = b'\x02' + bytes(3 * w) + raw = b''.join(row_noise() if y % 8 == 0 else row_same for y in range(h)) + return imagegen.png(w, h, raw=raw, level=6) + + +SIZES = { + 'S_320x200': lambda: (ROOT / 'tests/fixtures/sample.png').read_bytes(), + 'M_2000x1500': lambda: noisy_png(2000, 1500), + 'L_5000x4000': lambda: gradient_png(5000, 4000), +} +COUNTS = {'S_320x200': 120, 'M_2000x1500': 40, 'L_5000x4000': 16} +CONCURRENCY = [1, 2, 4] + + +def pct(values, p): + if not values: + return None + s = sorted(values) + return round(s[min(len(s) - 1, int(p * len(s)))], 3) + + +class LagSampler(threading.Thread): + def __init__(self, chat_id): + super().__init__(daemon=True) + self.chat_id = chat_id + self.samples = [] + self.stop = threading.Event() + + def run(self): + t0 = time.monotonic() + while not self.stop.is_set(): + try: + r = psql("SELECT count(*) FILTER (WHERE j.status IN ('queued','processing')), " + "COALESCE(extract(epoch FROM now() - min(j.created_at) FILTER (WHERE j.status IN ('queued','processing'))), 0) " + "FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE a.chat_id = '%s'" % self.chat_id) + active, oldest = r.split('|') + q = queues().get('attachment.process.v1', {}) + self.samples.append({'t': round(time.monotonic() - t0, 1), 'active': int(active), + 'mq_ready': q.get('ready'), 'mq_unacked': q.get('unacked'), + 'oldest_age_s': round(float(oldest), 2)}) + except Exception: + pass + self.stop.wait(0.5) + + +def burst(c, actor, chat, data, n, threads=8): + codes = [] + lock = threading.Lock() + + def one(i): + try: + r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=t1-%d.png' % i, [202, 503], + actor=actor, raw=data, headers={'Content-Type': 'image/png'}) + code = r['status'] + ra = r['headers'].get('Retry-After') + except Failure as exc: + code, ra = 'error:' + str(exc)[-40:], None + with lock: + codes.append((code, ra)) + + t0 = time.monotonic() + with ThreadPoolExecutor(threads) as pool: + list(pool.map(one, range(n))) + return codes, time.monotonic() - t0 + + +def wait_drained(chat_id, timeout=900): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + n = int(psql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id " + "WHERE a.chat_id = '%s' AND j.status IN ('queued','processing')" % chat_id)) + if n == 0: + return True + time.sleep(1) + return False + + +def job_stats(chat_id): + rows = psql("SELECT j.status, extract(epoch FROM j.created_at), extract(epoch FROM j.finished_at), j.attempt " + "FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE a.chat_id = '%s'" % chat_id) + jobs = [r.split('|') for r in rows.splitlines()] + ready = [(float(c), float(f)) for s, c, f, _ in jobs if s == 'ready'] + ttr = [f - c for c, f in ready] + span = (max(f for _, f in ready) - min(c for c, _ in ready)) if ready else 0 + return {'jobs': len(jobs), 'ready': len(ready), 'failed': sum(1 for j in jobs if j[0] == 'failed'), + 'retried_jobs': sum(1 for j in jobs if int(j[3]) > 1), + 'throughput_jobs_per_s': round(len(ready) / span, 2) if span else None, + 'pipeline_span_s': round(span, 2), + 'time_to_ready_s': {'p50': pct(ttr, .5), 'p95': pct(ttr, .95), 'max': round(max(ttr), 3) if ttr else None}} + + +def set_worker(concurrency): + compose('up', '-d', '--no-deps', '--wait', 'worker', env={'WORKER_CONCURRENCY': str(concurrency)}) + time.sleep(2) + + +def one_run(base_url, size, conc): + c = Client(base_url, timeout=60) + alice, _bob, _eve, chat = c.bootstrap() + data = SIZES[size]() + n = COUNTS[size] + lag = LagSampler(chat['id']) + with MemSampler('worker') as mem: + lag.start() + codes, upload_s = burst(c, alice, chat, data, n) + drained = wait_drained(chat['id']) + time.sleep(1) + lag.stop.set() + lag.join() + st = job_stats(chat['id']) + s = lag.samples + return {'size': size, 'image_bytes': len(data), 'worker_concurrency_prefetch': conc, 'uploads': n, + 'accepted_202': sum(1 for x, _ in codes if x == 202), + 'rejected': sum(1 for x, _ in codes if x != 202), + 'upload_burst_s': round(upload_s, 2), 'drained': drained, **st, + 'queue_lag': {'max_active_jobs': max((x['active'] for x in s), default=None), + 'max_mq_ready': max((x['mq_ready'] or 0 for x in s), default=None), + 'max_oldest_job_age_s': max((x['oldest_age_s'] for x in s), default=None)}, + 'worker_peak_rss_mib': round(mem.peak, 1), + 'lag_timeline': s[::max(1, len(s) // 40)]} + + +def cmd_matrix(a): + runs = [] + try: + for conc in CONCURRENCY: + set_worker(conc) + for size in SIZES: + r = one_run(a.base_url, size, conc) + print(json.dumps({k: r[k] for k in ('size', 'worker_concurrency_prefetch', 'throughput_jobs_per_s', + 'time_to_ready_s', 'worker_peak_rss_mib', 'queue_lag')}), flush=True) + runs.append(r) + finally: + set_worker(2) + return {'runs': runs} + + +def cmd_backpressure(a): + bound = 20 + compose('up', '-d', '--no-deps', '--wait', 'api', env={'MAX_ACTIVE_JOBS': str(bound)}) + set_worker(1) + try: + c = Client(a.base_url, timeout=60) + alice, _bob, _eve, chat = c.bootstrap() + data = SIZES['M_2000x1500']() + lag = LagSampler(chat['id']) + lag.start() + t0 = time.monotonic() + codes, upload_s = burst(c, alice, chat, data, 80) + drained = wait_drained(chat['id']) + drain_s = time.monotonic() - t0 + t1 = time.monotonic() + att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=after.png', 202, actor=alice, + raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(), headers={'Content-Type': 'image/png'}, schema='Attachment')['json'] + while True: + st = c.call('GET', '/api/v1/attachments/' + att['id'], actor=alice, schema='Attachment')['json']['status'] + if st in ('ready', 'failed'): + break + time.sleep(0.2) + recovery_ready_s = time.monotonic() - t1 + time.sleep(1) + lag.stop.set() + lag.join() + s = lag.samples + return {'max_active_jobs_bound': bound, 'worker_concurrency': 1, 'uploads': 80, + 'accepted_202': sum(1 for x, _ in codes if x == 202), + 'rejected_503': sum(1 for x, _ in codes if x == 503), + 'retry_after_header': sorted({ra for x, ra in codes if x == 503}), + 'other': [x for x, _ in codes if x not in (202, 503)], + 'upload_burst_s': round(upload_s, 2), 'drained': drained, 'drain_s': round(drain_s, 1), + 'max_active_observed': max((x['active'] for x in s), default=None), + 'jobs': job_stats(chat['id']), 'after_overload_upload_status': st, + 'after_overload_seconds_to_ready': round(recovery_ready_s, 2), + 'lag_timeline': s[::max(1, len(s) // 40)]} + finally: + compose('up', '-d', '--no-deps', '--wait', 'api') + set_worker(2) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument('command', choices=['matrix', 'backpressure']) + p.add_argument('--base-url', default='http://localhost:8080') + p.add_argument('--output') + a = p.parse_args() + env = {'started_at': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), + 'worker_limits': 'cpus=2.0 memory=768M GOMEMLIMIT=600MiB decode budget 384MiB', + 'host': 'Windows 11, Docker Desktop 29.7.2 (WSL2)'} + res = {'experiment': a.command, **env, **(cmd_matrix(a) if a.command == 'matrix' else cmd_backpressure(a))} + text = json.dumps(res, ensure_ascii=False, indent=1) + if a.output: + (ROOT / a.output).parent.mkdir(parents=True, exist_ok=True) + (ROOT / a.output).write_text(text + '\n', encoding='utf-8') + print(text[:3000]) + + +if __name__ == '__main__': + main() diff --git a/src/cmd/server/main.go b/src/cmd/server/main.go new file mode 100644 index 00000000..db7829b3 --- /dev/null +++ b/src/cmd/server/main.go @@ -0,0 +1,403 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "log/slog" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "sync/atomic" + "syscall" + "time" + + "mtusi-messenger/internal/api" + "mtusi-messenger/internal/auth" + "mtusi-messenger/internal/broker" + "mtusi-messenger/internal/cache" + "mtusi-messenger/internal/db" + "mtusi-messenger/internal/imaging" + "mtusi-messenger/internal/objstore" + "mtusi-messenger/internal/relay" + "mtusi-messenger/internal/store" + "mtusi-messenger/internal/worker" + + "golang.org/x/sync/semaphore" +) + +func getenv(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func getint(key string, def int) int { + v := os.Getenv(key) + if v == "" { + return def + } + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + fmt.Fprintf(os.Stderr, "invalid %s\n", key) + os.Exit(2) + } + return n +} + +func secret(key string) (string, error) { + if path := os.Getenv(key + "_FILE"); path != "" { + b, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read %s_FILE: %w", key, err) + } + return strings.TrimSpace(string(b)), nil + } + if v := os.Getenv(key); v != "" { + return v, nil + } + return "", fmt.Errorf("%s or %s_FILE is required", key, key) +} + +func main() { + log := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + if err := run(log); err != nil { + log.Error("fatal", "error", err.Error()) + os.Exit(1) + } +} + +func openDB(ctx context.Context) (*store.Store, func(), error) { + dsn, err := secret("DATABASE_URL") + if err != nil { + return nil, nil, err + } + pool, err := db.Open(ctx, dsn, db.PoolOptions{MaxConns: int32(getint("DB_MAX_CONNS", 10))}) + if err != nil { + return nil, nil, errors.New("database config is invalid") + } + return store.New(pool), pool.Close, nil +} + +func getduration(key string, def time.Duration) time.Duration { + v := os.Getenv(key) + if v == "" { + return def + } + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + fmt.Fprintf(os.Stderr, "invalid %s\n", key) + os.Exit(2) + } + return d +} + +func openObjects() (*objstore.Store, error) { return openObjectsAs("S3") } + +func openObjectsAs(prefix string) (*objstore.Store, error) { + access, err := secret(prefix + "_ACCESS_KEY_ID") + if err != nil { + return nil, err + } + secretKey, err := secret(prefix + "_SECRET_ACCESS_KEY") + if err != nil { + return nil, err + } + return objstore.New(objstore.Config{ + Endpoint: getenv("S3_ENDPOINT", "http://s3:9000"), + Region: getenv("S3_REGION", "us-east-1"), + Bucket: getenv("S3_BUCKET", "messenger-attachments"), + AccessKey: access, + SecretKey: secretKey, + }) +} + +func getbool(key string) bool { + v := strings.ToLower(os.Getenv(key)) + return v == "1" || v == "true" || v == "yes" +} + +func startRelay(ctx context.Context, st *store.Store, log *slog.Logger) (*relay.Relay, func(), error) { + url, err := secret("BROKER_URL") + if err != nil { + return nil, nil, err + } + pub := broker.NewPublisher(url) + rl := relay.New(st, pub, log, getduration("OUTBOX_POLL_INTERVAL", time.Second)) + rctx, cancel := context.WithCancel(ctx) + go rl.Run(rctx) + return rl, func() { cancel(); pub.Close() }, nil +} + +func runWorker(ctx context.Context, log *slog.Logger) error { + st, closeDB, err := openDB(ctx) + if err != nil { + return err + } + defer closeDB() + objects, err := openObjects() + if err != nil { + return err + } + url, err := secret("BROKER_URL") + if err != nil { + return err + } + budget := int64(getint("WORKER_DECODE_BUDGET_BYTES", 384<<20)) + cfg := worker.Config{ + BrokerURL: url, + Concurrency: getint("WORKER_CONCURRENCY", 2), + MaxAttempts: getint("JOB_MAX_ATTEMPTS", 5), + JobTimeout: time.Duration(getint("JOB_TIMEOUT_SECONDS", 60)) * time.Second, + Lease: time.Duration(getint("JOB_LEASE_SECONDS", 30)) * time.Second, + BackoffBase: getduration("JOB_BACKOFF_BASE", time.Second), + BackoffMax: getduration("JOB_BACKOFF_MAX", time.Minute), + StagingRetain: time.Duration(getint("STAGING_RETENTION_SECONDS", 86400)) * time.Second, + PendingGrace: getduration("PENDING_UPLOAD_GRACE", 15*time.Minute), + FaultInjection: getbool("FAULT_INJECTION"), + SweepEvery: getduration("SWEEP_INTERVAL", 5*time.Second), + Limits: imaging.Limits{ + MaxBytes: int64(getint("MAX_UPLOAD_BYTES", 10485760)), + MaxPixels: int64(getint("MAX_IMAGE_PIXELS", 40000000)), + Budget: semaphore.NewWeighted(budget), + BudgetSize: budget, + }, + } + if cfg.Lease < 6*time.Second { + return errors.New("JOB_LEASE_SECONDS must be >= 6") + } + wctx, stop := signal.NotifyContext(ctx, syscall.SIGTERM, syscall.SIGINT) + defer stop() + if !getbool("WORKER_NO_RELAY") { + _, stopRelay, err := startRelay(wctx, st, log) + if err != nil { + return err + } + defer stopRelay() + } + log.Info("worker started", "concurrency", cfg.Concurrency, "max_attempts", cfg.MaxAttempts, + "job_timeout_s", int(cfg.JobTimeout.Seconds()), "lease_s", int(cfg.Lease.Seconds()), + "fault_injection", cfg.FaultInjection, "build", getenv("APP_BUILD", "dev")) + return worker.New(cfg, st, objects, log).Run(wctx) +} + +func runReplay(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("replay", flag.ContinueOnError) + actor := fs.String("actor", "operator", "who requested the replay (audit)") + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + args = append(args[1:], args[0]) + } + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 1 { + return errors.New("usage: replay [--actor name]") + } + st, closeDB, err := openDB(ctx) + if err != nil { + return err + } + defer closeDB() + jobID, attempt, err := st.ReplayFailed(ctx, fs.Arg(0), *actor, getint("JOB_MAX_ATTEMPTS", 5)) + switch { + case errors.Is(err, store.ErrJobAlreadyOK): + fmt.Println(`{"result":"noop","reason":"job already ready; replay refused"}`) + return nil + case errors.Is(err, store.ErrJobNotFailed): + fmt.Println(`{"result":"noop","reason":"job is queued/processing, not failed"}`) + return nil + case errors.Is(err, store.ErrStagingGone): + fmt.Println(`{"result":"refused","reason":"staging original removed by retention"}`) + return nil + case err != nil: + return err + } + out, _ := json.Marshal(map[string]any{"result": "replayed", "job_id": jobID, "attempt": attempt}) + fmt.Println(string(out)) + return nil +} + +func run(log *slog.Logger) error { + ctx := context.Background() + + if len(os.Args) > 1 && os.Args[1] == "migrate" { + dsn, err := secret("DATABASE_URL") + if err != nil { + return err + } + pool, err := db.Open(ctx, dsn, db.PoolOptions{MaxConns: 2}) + if err != nil { + return errors.New("database config is invalid") + } + defer pool.Close() + mctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + return db.Migrate(mctx, pool, log) + } + if len(os.Args) > 1 && os.Args[1] == "s3-init" { + objects, err := openObjectsAs("S3_ADMIN") + if err != nil { + return err + } + var created bool + for attempt := 1; ; attempt++ { + bctx, cancel := context.WithTimeout(ctx, 5*time.Second) + created, err = objects.EnsureBucket(bctx, getenv("S3_REGION", "us-east-1")) + cancel() + if err == nil || attempt == 30 { + break + } + log.Warn("s3 not ready, retrying", "attempt", attempt, "error", err.Error()) + time.Sleep(2 * time.Second) + } + if err != nil { + return err + } + log.Info("bucket ready", "bucket", objects.Bucket(), "created", created) + return nil + } + if len(os.Args) > 1 && os.Args[1] == "worker" { + return runWorker(ctx, log) + } + if len(os.Args) > 1 && os.Args[1] == "replay" { + return runReplay(ctx, os.Args[2:]) + } + + st, closeDB, err := openDB(ctx) + if err != nil { + return err + } + defer closeDB() + + ttl := time.Duration(getint("SESSION_TTL_SECONDS", 3600)) * time.Second + cacheMaxTTL := time.Duration(getint("SESSION_CACHE_MAX_TTL_SECONDS", 60)) * time.Second + + var sessCache auth.SessionCache + switch mode := getenv("SESSION_STORE", "postgres"); mode { + case "postgres": + case "valkey", "redis": + url, err := secret("CACHE_URL") + if err != nil { + return err + } + vk, err := cache.NewValkey(url) + if err != nil { + return errors.New("cache config is invalid") + } + defer vk.Close() + sessCache = vk + default: + return fmt.Errorf("unknown SESSION_STORE %q (postgres|valkey|redis)", mode) + } + + hasher := auth.NewHasher(getint("PASSWORD_HASH_CONCURRENCY", 4)) + sessions := auth.NewSessions(st, sessCache, ttl, cacheMaxTTL, log) + + instanceID := getenv("INSTANCE_ID", "") + if instanceID == "" { + h, err := os.Hostname() + if err != nil || h == "" { + return errors.New("cannot determine instance id; set INSTANCE_ID") + } + instanceID = h + } + idemTTL := time.Duration(getint("IDEMPOTENCY_TTL_SECONDS", 86400)) * time.Second + objects, err := openObjects() + if err != nil { + return err + } + maxUpload := int64(getint("MAX_UPLOAD_BYTES", 10485760)) + if maxUpload > 10485760 { + return errors.New("MAX_UPLOAD_BYTES above the 10 MiB contract limit") + } + drainDelay := time.Duration(getint("SHUTDOWN_DRAIN_MS", 3000)) * time.Millisecond + + rl, stopRelay, err := startRelay(ctx, st, log) + if err != nil { + return err + } + defer stopRelay() + + ready := &atomic.Bool{} + ready.Store(true) + + host := getenv("APP_HOST", "0.0.0.0") + port := getenv("APP_PORT", "8080") + addr := host + ":" + port + srv := api.NewServer(st, objects, sessions, hasher, log, ready, api.Options{ + InstanceID: instanceID, IdempotencyTTL: idemTTL, + MaxUploadBytes: maxUpload, + MaxImagePixels: int64(getint("MAX_IMAGE_PIXELS", 40000000)), + UploadConcurrency: int64(getint("UPLOAD_CONCURRENCY", 8)), + UploadTimeout: getduration("UPLOAD_TIMEOUT", 60*time.Second), + S3Timeout: getduration("S3_TIMEOUT", 15*time.Second), + Relay: rl, + MaxActiveJobs: getint("MAX_ACTIVE_JOBS", 500), + PipelineVersion: imaging.PipelineVersion, + MaxAttempts: getint("JOB_MAX_ATTEMPTS", 5), + FaultInjection: getbool("FAULT_INJECTION"), + }) + httpSrv := &http.Server{ + Addr: addr, + Handler: srv.Handler(), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 90 * time.Second, + WriteTimeout: 90 * time.Second, + IdleTimeout: 60 * time.Second, + } + + bgCtx, stopBg := context.WithCancel(ctx) + defer stopBg() + go func() { + t := time.NewTicker(10 * time.Minute) + defer t.Stop() + for { + select { + case <-bgCtx.Done(): + return + case <-t.C: + c, cancel := context.WithTimeout(bgCtx, 10*time.Second) + if n, err := st.PurgeSessions(c, 24*time.Hour); err == nil && n > 0 { + log.Info("expired sessions purged", "count", n) + } + if n, err := st.PurgeIdempotency(c, idemTTL); err == nil && n > 0 { + log.Info("expired idempotency keys purged", "count", n) + } + cancel() + } + } + }() + + errCh := make(chan error, 1) + go func() { errCh <- httpSrv.ListenAndServe() }() + log.Info("server started", "addr", addr, "pid", os.Getpid(), "instance_id", instanceID, "build", getenv("APP_BUILD", "dev"), "s3_bucket", objects.Bucket(), + "session_store", getenv("SESSION_STORE", "postgres"), "session_ttl_s", int(ttl.Seconds())) + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) + + select { + case err := <-errCh: + if err != nil && !errors.Is(err, http.ErrServerClosed) { + return err + } + case sig := <-stop: + log.Info("shutdown signal received", "signal", sig.String()) + ready.Store(false) + time.Sleep(drainDelay) + httpSrv.SetKeepAlivesEnabled(false) + sctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := httpSrv.Shutdown(sctx); err != nil { + log.Warn("graceful shutdown exceeded budget, closing", "error", err.Error()) + _ = httpSrv.Close() + } + log.Info("server stopped cleanly") + } + return nil +} diff --git a/src/go.mod b/src/go.mod new file mode 100644 index 00000000..b998ea11 --- /dev/null +++ b/src/go.mod @@ -0,0 +1,35 @@ +module mtusi-messenger + +go 1.23 + +require ( + github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.7.2 + github.com/minio/minio-go/v7 v7.0.82 + github.com/redis/go-redis/v9 v9.7.0 + golang.org/x/crypto v0.31.0 + golang.org/x/sync v0.10.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.8.1 // indirect + github.com/gen2brain/webp v0.5.2 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.3 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/rabbitmq/amqp091-go v1.10.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tetratelabs/wazero v1.8.1 // indirect + golang.org/x/image v0.23.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect +) diff --git a/src/go.sum b/src/go.sum new file mode 100644 index 00000000..7fb69472 --- /dev/null +++ b/src/go.sum @@ -0,0 +1,72 @@ +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.8.1 h1:sdRKd6plj7KYW33EH5As6YKfe8m9zbN9JMrOjNVF/BE= +github.com/ebitengine/purego v0.8.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/gen2brain/webp v0.5.2 h1:aYdjbU/2L98m+bqUdkYMOIY93YC+EN3HuZLMaqgMD9U= +github.com/gen2brain/webp v0.5.2/go.mod h1:Nb3xO5sy6MeUAHhru9H3GT7nlOQO5dKRNNlE92CZrJw= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI= +github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.82 h1:tWfICLhmp2aFPXL8Tli0XDTHj2VB/fNf0PC1f/i1gRo= +github.com/minio/minio-go/v7 v7.0.82/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= +github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tetratelabs/wazero v1.8.1 h1:NrcgVbWfkWvVc4UtT4LRLDf91PsOzDzefMdwhLfA550= +github.com/tetratelabs/wazero v1.8.1/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/image v0.23.0 h1:HseQ7c2OpPKTPVzNjG5fwJsOTCiiwS4QdsYi5XU6H68= +golang.org/x/image v0.23.0/go.mod h1:wJJBTdLfCCf3tiHa1fNxpZmUI4mmoZvwMCPP0ddoNKY= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src/internal/api/attachments.go b/src/internal/api/attachments.go new file mode 100644 index 00000000..d8ce7f24 --- /dev/null +++ b/src/internal/api/attachments.go @@ -0,0 +1,334 @@ +package api + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "mime" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/google/uuid" + + "mtusi-messenger/internal/apperr" + "mtusi-messenger/internal/model" + "mtusi-messenger/internal/objstore" + "mtusi-messenger/internal/store" +) + +const ( + mimePNG = "image/png" + mimeJPEG = "image/jpeg" + + maxDrainBytes = 64 << 20 +) + +func (s *Server) objectError(w http.ResponseWriter, r *http.Request, err error) { + s.log.Warn("object storage unavailable", "request_id", requestID(r), "error", err.Error()) + apperr.Write(w, requestID(r), apperr.Unavailable("attachment storage temporarily unavailable")) +} + +func validFilename(name string) bool { + if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || name == "." || name == ".." { + return false + } + if n := runeLen(name); n < 1 || n > 255 { + return false + } + for _, c := range name { + switch { + case c == '/', c == '\\', c == 0, unicode.IsControl(c): + return false + case c >= 0x202a && c <= 0x202e, c >= 0x2066 && c <= 0x2069: + return false + } + } + return true +} + +func sniffImage(data []byte) string { + switch { + case bytes.HasPrefix(data, []byte("\x89PNG\r\n\x1a\n")): + return mimePNG + case bytes.HasPrefix(data, []byte{0xff, 0xd8, 0xff}): + return mimeJPEG + } + return "" +} + +type imageInfo struct { + contentType string + width, height int +} + +func (s *Server) inspectImage(data []byte, declared string) (imageInfo, *apperr.AppError) { + detected := sniffImage(data) + if detected == "" { + return imageInfo{}, apperr.UnsupportedMedia("content is not a PNG or JPEG image") + } + if detected != declared { + return imageInfo{}, apperr.UnsupportedMedia("Content-Type does not match the image content") + } + cfg, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || cfg.Width <= 0 || cfg.Height <= 0 || (format != "png" && format != "jpeg") { + return imageInfo{}, apperr.UnsupportedMedia("image header is invalid") + } + pixels := int64(cfg.Width) * int64(cfg.Height) + if pixels > s.maxPixels { + return imageInfo{}, apperr.PayloadTooLarge("image exceeds " + strconv.FormatInt(s.maxPixels, 10) + " pixels") + } + return imageInfo{contentType: detected, width: cfg.Width, height: cfg.Height}, nil +} + +func readBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, *apperr.AppError) { + var buf bytes.Buffer + if r.ContentLength > 0 { + buf.Grow(int(r.ContentLength)) + } + _, err := buf.ReadFrom(http.MaxBytesReader(w, r.Body, limit)) + var maxErr *http.MaxBytesError + switch { + case errors.As(err, &maxErr): + return nil, apperr.PayloadTooLarge("attachment exceeds " + strconv.FormatInt(limit, 10) + " bytes") + case err != nil: + return nil, apperr.InvalidRequest("request body could not be read") + case buf.Len() == 0: + return nil, apperr.InvalidRequest("request body is empty") + } + return buf.Bytes(), nil +} + +func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + methodNotAllowed(w, r) + return + } + body := r.Body + defer func() { _, _ = io.CopyN(io.Discard, body, maxDrainBytes) }() + + id := requestID(r) + chatID, ok := s.chatAccess(w, r) + if !ok { + return + } + query, err := url.ParseQuery(r.URL.RawQuery) + if err != nil || len(query["filename"]) != 1 { + apperr.Write(w, id, apperr.InvalidRequest("exactly one filename query parameter is required")) + return + } + filename := query.Get("filename") + if !validFilename(filename) { + apperr.Write(w, id, apperr.InvalidRequest("filename must be 1..255 code points without path separators or control characters")) + return + } + media, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")) + if media != mimePNG && media != mimeJPEG { + apperr.Write(w, id, apperr.UnsupportedMedia("Content-Type must be image/png or image/jpeg")) + return + } + if r.ContentLength > s.maxUpload { + apperr.Write(w, id, apperr.PayloadTooLarge("attachment exceeds "+strconv.FormatInt(s.maxUpload, 10)+" bytes")) + return + } + + if s.maxActiveJobs > 0 { + n, err := s.store.ActiveJobs(r.Context()) + if err != nil { + s.storeError(w, r, err) + return + } + if n >= s.maxActiveJobs { + w.Header().Set("Retry-After", "5") + apperr.Write(w, id, apperr.Unavailable("attachment processing queue is full, retry later")) + return + } + } + + if err := s.uploadSem.Acquire(r.Context(), 1); err != nil { + apperr.Write(w, id, apperr.Unavailable("too many concurrent uploads, retry later")) + return + } + defer s.uploadSem.Release(1) + + data, aerr := readBody(w, r, s.maxUpload) + if aerr != nil { + apperr.Write(w, id, aerr) + return + } + info, aerr := s.inspectImage(data, media) + if aerr != nil { + apperr.Write(w, id, aerr) + return + } + sum := sha256.Sum256(data) + attID := uuid.NewString() + att := &model.Attachment{ + ID: attID, ChatID: chatID, OwnerID: currentUser(r).ID, Filename: filename, + ContentType: info.contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(sum[:]), + Status: "queued", Variants: []model.Variant{}, + + ObjectKey: "attachments/" + chatID + "/" + attID, + } + stagingKey := "staging/" + attID + + if err := s.store.InsertPendingAttachment(r.Context(), att, info.width, info.height, stagingKey); err != nil { + if errors.Is(err, store.ErrNotFound) { + apperr.Write(w, id, apperr.NotFound("chat not found or not accessible")) + return + } + s.storeError(w, r, err) + return + } + putCtx, cancel := context.WithTimeout(r.Context(), s.s3Timeout) + err = s.objects.Put(putCtx, stagingKey, data, att.ContentType) + cancel() + if err != nil { + go s.compensateUpload(att, stagingKey, id) + s.objectError(w, r, err) + return + } + _, updated, err := s.store.QueueAttachment(r.Context(), att.ID, s.pipelineVersion, s.maxAttempts, s.maxActiveJobs) + if err != nil { + go s.compensateUpload(att, stagingKey, id) + if errors.Is(err, store.ErrQueueFull) { + w.Header().Set("Retry-After", "5") + apperr.Write(w, id, apperr.Unavailable("attachment processing queue is full, retry later")) + return + } + if errors.Is(err, store.ErrNotFound) { + apperr.Write(w, id, apperr.Unavailable("upload expired, retry")) + return + } + s.storeError(w, r, err) + return + } + if s.faultInjection && strings.HasPrefix(filename, "fault-api-after-commit") { + s.log.Warn("FAULT: exit after COMMIT before publish/response", "attachment_id", att.ID) + os.Exit(42) + } + if s.relay != nil { + s.relay.Kick() + } + att.UpdatedAt = model.NewTime(updated) + writeJSON(w, r, http.StatusAccepted, att) +} + +func (s *Server) compensateUpload(att *model.Attachment, stagingKey, reqID string) { + dctx, dcancel := context.WithTimeout(context.Background(), 3*time.Second) + defer dcancel() + deleted, err := s.store.DeletePendingAttachment(dctx, att.ID) + if err != nil || !deleted { + if err != nil { + s.log.Warn("upload compensation deferred to reconciler", "request_id", reqID, + "attachment_id", att.ID, "stage", "delete_row") + } + return + } + octx, ocancel := context.WithTimeout(context.Background(), 3*time.Second) + defer ocancel() + if err := s.objects.Remove(octx, stagingKey); err != nil { + s.log.Warn("upload compensation deferred to reconciler", "request_id", reqID, + "attachment_id", att.ID, "stage", "remove_object") + } +} + +func (s *Server) attachmentForRequest(w http.ResponseWriter, r *http.Request) (*model.Attachment, bool) { + id := requestID(r) + attID, err := parseUUID(r.PathValue("attachment_id")) + if err != nil { + apperr.Write(w, id, apperr.InvalidRequest("attachment_id must be a UUID")) + return nil, false + } + att, err := s.store.AttachmentForUser(r.Context(), attID, currentUser(r).ID) + if errors.Is(err, store.ErrNotFound) { + apperr.Write(w, id, apperr.NotFound("attachment not found or not accessible")) + return nil, false + } + if err != nil { + s.storeError(w, r, err) + return nil, false + } + return att, true +} + +func (s *Server) getAttachment(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + methodNotAllowed(w, r) + return + } + if att, ok := s.attachmentForRequest(w, r); ok { + writeJSON(w, r, http.StatusOK, att) + } +} + +func (s *Server) downloadAttachment(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + methodNotAllowed(w, r) + return + } + id := requestID(r) + att, ok := s.attachmentForRequest(w, r) + if !ok { + return + } + variant := r.URL.Query().Get("variant") + switch variant { + case "", "original", "thumbnail", "crop", "optimized": + default: + apperr.Write(w, id, apperr.InvalidRequest("unknown variant")) + return + } + if att.Status != "ready" { + apperr.Write(w, id, apperr.Conflict("attachment is "+att.Status+", content is not available yet")) + return + } + key, contentType, etag := att.ObjectKey, att.ContentType, att.SHA256 + if variant != "" && variant != "original" { + found := false + for _, v := range att.Variants { + if v.Name == variant { + key, contentType, etag, found = v.ObjectKey, v.ContentType, v.SHA256, true + } + } + if !found { + apperr.Write(w, id, apperr.NotFound("variant not found")) + return + } + } + ctx, cancel := context.WithTimeout(r.Context(), s.s3Timeout) + defer cancel() + obj, size, err := s.objects.Get(ctx, key) + if objstore.IsNotFound(err) { + s.log.Error("attachment object missing in S3", "request_id", id, "attachment_id", att.ID) + apperr.Write(w, id, apperr.NotFound("attachment content is not available")) + return + } + if err != nil { + s.objectError(w, r, err) + return + } + defer obj.Close() + h := w.Header() + h.Set("Content-Type", contentType) + h.Set("Content-Length", strconv.FormatInt(size, 10)) + h.Set("X-Content-Type-Options", "nosniff") + h.Set("Cache-Control", "private, no-store") + h.Set("Content-Disposition", "inline; filename*=UTF-8''"+url.PathEscape(att.Filename)) + h.Set("ETag", `"`+etag+`"`) + w.WriteHeader(http.StatusOK) + if _, err := io.Copy(w, obj); err != nil { + s.log.Warn("download interrupted", "request_id", id, "attachment_id", att.ID, "error", err.Error()) + } +} diff --git a/src/internal/api/handlers.go b/src/internal/api/handlers.go new file mode 100644 index 00000000..9d0338ba --- /dev/null +++ b/src/internal/api/handlers.go @@ -0,0 +1,431 @@ +package api + +import ( + "context" + "encoding/base64" + "errors" + "net/http" + "strings" + "time" + "unicode/utf8" + + "mtusi-messenger/internal/apperr" + "mtusi-messenger/internal/auth" + "mtusi-messenger/internal/model" + "mtusi-messenger/internal/store" +) + +func (s *Server) storeError(w http.ResponseWriter, r *http.Request, err error) { + id := requestID(r) + if store.IsUnavailable(err) || errors.Is(err, auth.ErrCacheUnavailable) { + s.log.Warn("dependency unavailable", "request_id", id, "error", err.Error()) + apperr.Write(w, id, apperr.Unavailable("storage temporarily unavailable")) + return + } + s.log.Error("unexpected storage error", "request_id", id, "error", err.Error()) + apperr.Write(w, id, apperr.Internal("internal error")) +} + +func (s *Server) healthLive(w http.ResponseWriter, r *http.Request) { + writeJSON(w, r, http.StatusOK, model.Health{Status: "ok"}) +} + +func (s *Server) healthReady(w http.ResponseWriter, r *http.Request) { + if !s.ready.Load() { + apperr.Write(w, requestID(r), apperr.Unavailable("service is shutting down")) + return + } + ctx, cancel := context.WithTimeout(r.Context(), time.Second) + defer cancel() + if err := s.store.Ping(ctx); err != nil { + apperr.Write(w, requestID(r), apperr.Unavailable("database is not reachable")) + return + } + writeJSON(w, r, http.StatusOK, model.Health{Status: "ok"}) +} + +type createUserReq struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` +} + +func validPassword(p string) bool { + n := utf8.RuneCountInString(p) + return utf8.ValidString(p) && n >= 12 && n <= 128 +} + +func (s *Server) usersRoute(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + methodNotAllowed(w, r) + return + } + id := requestID(r) + var req createUserReq + if aerr := decodeJSON(w, r, &req); aerr != nil { + apperr.Write(w, id, aerr) + return + } + if !usernameRe.MatchString(req.Username) { + apperr.Write(w, id, apperr.InvalidRequest("username must match ^[a-z0-9_]{3,32}$")) + return + } + if !validText(req.DisplayName, 100) { + apperr.Write(w, id, apperr.InvalidRequest("display_name must be 1..100 non-space code points")) + return + } + if !validPassword(req.Password) { + apperr.Write(w, id, apperr.InvalidRequest("password must be 12..128 code points")) + return + } + hash, err := s.hasher.Hash(r.Context(), req.Password) + if err != nil { + s.storeError(w, r, err) + return + } + u, err := s.store.CreateUser(r.Context(), req.Username, req.DisplayName, hash) + if errors.Is(err, store.ErrUsernameTaken) { + apperr.Write(w, id, apperr.Conflict("username is already taken")) + return + } + if err != nil { + s.storeError(w, r, err) + return + } + writeJSON(w, r, http.StatusCreated, u) +} + +func (s *Server) me(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + methodNotAllowed(w, r) + return + } + writeJSON(w, r, http.StatusOK, currentUser(r)) +} + +const basicChallenge = `Basic realm="messenger", charset="UTF-8"` + +func unauthorizedBasic(w http.ResponseWriter, r *http.Request) { + w.Header().Set("WWW-Authenticate", basicChallenge) + apperr.Write(w, requestID(r), apperr.Unauthorized("invalid username or password")) +} + +func parseBasic(r *http.Request) (string, string, bool) { + scheme, value, ok := strings.Cut(r.Header.Get("Authorization"), " ") + if !ok || !strings.EqualFold(scheme, "Basic") { + return "", "", false + } + raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(value)) + if err != nil || !utf8.Valid(raw) { + return "", "", false + } + user, pass, ok := strings.Cut(string(raw), ":") + if !ok || !usernameRe.MatchString(user) || !validPassword(pass) { + return "", "", false + } + return user, pass, true +} + +func (s *Server) createSession(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + methodNotAllowed(w, r) + return + } + username, password, ok := parseBasic(r) + if !ok { + unauthorizedBasic(w, r) + return + } + user, hash, err := s.store.Credentials(r.Context(), username) + if err != nil && !errors.Is(err, store.ErrNotFound) { + s.storeError(w, r, err) + return + } + + valid, err := s.hasher.Verify(r.Context(), password, hash) + if err != nil { + s.storeError(w, r, err) + return + } + if !valid { + unauthorizedBasic(w, r) + return + } + token, expires, err := s.sessions.Create(r.Context(), user) + if err != nil { + s.storeError(w, r, err) + return + } + writeJSON(w, r, http.StatusCreated, model.Session{ + Token: token, TokenType: "Bearer", ExpiresAt: model.NewTime(expires), User: *user, + }) +} + +func (s *Server) revokeSession(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + methodNotAllowed(w, r) + return + } + err := s.sessions.Revoke(r.Context(), currentToken(r)) + if errors.Is(err, auth.ErrInvalidSession) { + unauthorizedBearer(w, r) + return + } + if err != nil { + s.storeError(w, r, err) + return + } + w.Header().Set("X-Request-Id", requestID(r)) + w.WriteHeader(http.StatusNoContent) +} + +type createChatReq struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` +} + +func (s *Server) chatsRoute(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodPost: + s.createChat(w, r) + case http.MethodGet: + s.listChats(w, r) + default: + methodNotAllowed(w, r) + } +} + +func (s *Server) createChat(w http.ResponseWriter, r *http.Request) { + id := requestID(r) + var req createChatReq + if aerr := decodeJSON(w, r, &req); aerr != nil { + apperr.Write(w, id, aerr) + return + } + if !validText(req.Title, 100) { + apperr.Write(w, id, apperr.InvalidRequest("title must be 1..100 non-space code points")) + return + } + if len(req.MemberIDs) < 1 || len(req.MemberIDs) > 100 { + apperr.Write(w, id, apperr.InvalidRequest("member_ids must contain 1..100 items")) + return + } + seen := make(map[string]bool, len(req.MemberIDs)) + for _, raw := range req.MemberIDs { + uid, err := parseUUID(raw) + if err != nil { + apperr.Write(w, id, apperr.InvalidRequest("member_ids must be UUID strings")) + return + } + if seen[uid] { + apperr.Write(w, id, apperr.InvalidRequest("duplicate member_id")) + return + } + seen[uid] = true + } + caller := currentUser(r) + if !seen[caller.ID] { + apperr.Write(w, id, apperr.InvalidRequest("creator must be included in member_ids")) + return + } + + chat, err := s.store.CreateChat(r.Context(), caller.ID, req.Title, req.MemberIDs) + if errors.Is(err, store.ErrUnknownMember) { + apperr.Write(w, id, apperr.InvalidRequest("unknown member_id")) + return + } + if err != nil { + s.storeError(w, r, err) + return + } + writeJSON(w, r, http.StatusCreated, chat) +} + +func (s *Server) listChats(w http.ResponseWriter, r *http.Request) { + id := requestID(r) + caller := currentUser(r) + limit, cur, aerr := pagingParams(r, caller.ID) + if aerr != nil { + apperr.Write(w, id, aerr) + return + } + items, more, err := s.store.ListChats(r.Context(), caller.ID, limit, cur) + if err != nil { + s.storeError(w, r, err) + return + } + page := model.Page[*model.Chat]{Items: items} + if more && len(items) > 0 { + last := items[len(items)-1] + encoded := store.EncodeCursor(store.Cursor{Scope: caller.ID, TS: last.CreatedAt.Time, ID: last.ID}) + page.NextCursor = &encoded + } + writeJSON(w, r, http.StatusOK, page) +} + +func (s *Server) chatRoute(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + methodNotAllowed(w, r) + return + } + id := requestID(r) + chatID, err := parseUUID(r.PathValue("chat_id")) + if err != nil { + apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID")) + return + } + chat, err := s.store.ChatForUser(r.Context(), chatID, currentUser(r).ID) + if errors.Is(err, store.ErrNotFound) { + apperr.Write(w, id, apperr.NotFound("chat not found or not accessible")) + return + } + if err != nil { + s.storeError(w, r, err) + return + } + writeJSON(w, r, http.StatusOK, chat) +} + +type createMessageReq struct { + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` +} + +func (s *Server) messagesRoute(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodPost: + s.createMessage(w, r) + case http.MethodGet: + s.listMessages(w, r) + default: + methodNotAllowed(w, r) + } +} + +func (s *Server) chatAccess(w http.ResponseWriter, r *http.Request) (string, bool) { + id := requestID(r) + chatID, err := parseUUID(r.PathValue("chat_id")) + if err != nil { + apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID")) + return "", false + } + ok, err := s.store.IsMember(r.Context(), chatID, currentUser(r).ID) + if err != nil { + s.storeError(w, r, err) + return "", false + } + if !ok { + apperr.Write(w, id, apperr.NotFound("chat not found or not accessible")) + return "", false + } + return chatID, true +} + +func (s *Server) createMessage(w http.ResponseWriter, r *http.Request) { + id := requestID(r) + chatID, err := parseUUID(r.PathValue("chat_id")) + if err != nil { + apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID")) + return + } + + reject := func(aerr *apperr.AppError) { + if _, ok := s.chatAccess(w, r); !ok { + return + } + apperr.Write(w, id, aerr) + } + key := r.Header.Get("Idempotency-Key") + if !idempotencyKeyRe.MatchString(key) { + reject(apperr.InvalidRequest("Idempotency-Key header is required: [A-Za-z0-9._:-]{1,128}")) + return + } + var req createMessageReq + if aerr := decodeJSON(w, r, &req); aerr != nil { + reject(aerr) + return + } + if req.AttachmentIDs == nil { + req.AttachmentIDs = []string{} + } + if len(req.AttachmentIDs) > 10 { + reject(apperr.InvalidRequest("attachment_ids must contain at most 10 items")) + return + } + seen := make(map[string]bool, len(req.AttachmentIDs)) + for _, raw := range req.AttachmentIDs { + aid, err := parseUUID(raw) + if err != nil { + reject(apperr.InvalidRequest("attachment_ids must be UUID strings")) + return + } + if seen[aid] { + reject(apperr.InvalidRequest("duplicate attachment_id")) + return + } + seen[aid] = true + } + if len(req.AttachmentIDs) == 0 && !validText(req.Text, 4000) { + reject(apperr.InvalidRequest("text must be 1..4000 non-space code points when there are no attachments")) + return + } + if !utf8.ValidString(req.Text) || runeLen(req.Text) > 4000 { + reject(apperr.InvalidRequest("text must be at most 4000 code points")) + return + } + msg, replayed, err := s.store.AddMessage(r.Context(), chatID, currentUser(r).ID, req.Text, req.AttachmentIDs, + key, messageFingerprint(req), s.idempotencyTTL) + if errors.Is(err, store.ErrIdempotencyConflict) { + apperr.Write(w, id, apperr.Conflict("Idempotency-Key was already used with a different request")) + return + } + if errors.Is(err, store.ErrNotFound) { + apperr.Write(w, id, apperr.NotFound("chat not found or not accessible")) + return + } + if errors.Is(err, store.ErrAttachmentFailed) { + apperr.Write(w, id, apperr.Conflict("a failed attachment cannot be attached")) + return + } + if errors.Is(err, store.ErrAttachmentNotFound) { + apperr.Write(w, id, apperr.NotFound("attachment not found in this chat")) + return + } + if err != nil { + s.storeError(w, r, err) + return + } + if replayed { + w.Header().Set("Idempotent-Replayed", "true") + } + writeJSON(w, r, http.StatusCreated, msg) +} + +func (s *Server) listMessages(w http.ResponseWriter, r *http.Request) { + id := requestID(r) + chatID, ok := s.chatAccess(w, r) + if !ok { + return + } + limit, cur, aerr := pagingParams(r, chatID) + if aerr != nil { + apperr.Write(w, id, aerr) + return + } + items, more, err := s.store.ListMessages(r.Context(), chatID, limit, cur) + if err != nil { + s.storeError(w, r, err) + return + } + page := model.Page[*model.Message]{Items: items} + if more && len(items) > 0 { + last := items[len(items)-1] + encoded := store.EncodeCursor(store.Cursor{Scope: chatID, TS: last.CreatedAt.Time, ID: last.ID}) + page.NextCursor = &encoded + } + writeJSON(w, r, http.StatusOK, page) +} + +func (s *Server) notFound(w http.ResponseWriter, r *http.Request) { + apperr.Write(w, requestID(r), apperr.NotFound("route not found")) +} diff --git a/src/internal/api/helpers.go b/src/internal/api/helpers.go new file mode 100644 index 00000000..4ed2c6da --- /dev/null +++ b/src/internal/api/helpers.go @@ -0,0 +1,110 @@ +package api + +import ( + "crypto/sha256" + "encoding/json" + "errors" + "io" + "mime" + "net/http" + "regexp" + "strconv" + "strings" + "unicode/utf8" + + "github.com/google/uuid" + + "mtusi-messenger/internal/apperr" + "mtusi-messenger/internal/store" +) + +const maxBodyBytes = 1 << 20 + +var ( + usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) +) + +func messageFingerprint(req createMessageReq) []byte { + b, _ := json.Marshal(req) + sum := sha256.Sum256(b) + return sum[:] +} + +func parseUUID(s string) (string, error) { + u, err := uuid.Parse(s) + if err != nil || u.String() != s { + return "", errors.New("invalid uuid") + } + return u.String(), nil +} + +func runeLen(s string) int { return utf8.RuneCountInString(s) } + +func validText(s string, max int) bool { + n := runeLen(s) + return n >= 1 && n <= max && strings.TrimSpace(s) != "" +} + +func decodeJSON(w http.ResponseWriter, r *http.Request, dst any) *apperr.AppError { + media, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")) + if media != "application/json" { + return apperr.UnsupportedMedia("Content-Type must be application/json") + } + r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes) + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + return apperr.PayloadTooLarge("request body too large") + } + return apperr.InvalidRequest("malformed JSON body") + } + var extra json.RawMessage + if err := dec.Decode(&extra); err != io.EOF { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + return apperr.PayloadTooLarge("request body too large") + } + return apperr.InvalidRequest("malformed JSON body") + } + return nil +} + +func pagingParams(r *http.Request, scope string) (int, *store.Cursor, *apperr.AppError) { + limit := 50 + if v := r.URL.Query().Get("limit"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 1 || n > 100 { + return 0, nil, apperr.InvalidRequest("limit must be an integer in 1..100") + } + limit = n + } + var cur *store.Cursor + if v := r.URL.Query().Get("cursor"); v != "" { + c, err := store.DecodeCursor(v) + if err == nil { + _, err = parseUUID(c.ID) + } + if err != nil { + return 0, nil, apperr.InvalidRequest("invalid cursor") + } + if c.Scope != scope { + return 0, nil, apperr.InvalidRequest("cursor does not belong to this resource") + } + cur = &c + } + return limit, cur, nil +} + +func writeJSON(w http.ResponseWriter, r *http.Request, status int, payload any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("X-Request-Id", requestID(r)) + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(payload) +} + +func methodNotAllowed(w http.ResponseWriter, r *http.Request) { + apperr.Write(w, requestID(r), apperr.New(http.StatusMethodNotAllowed, apperr.CodeInvalidRequest, "method not allowed")) +} diff --git a/src/internal/api/middleware.go b/src/internal/api/middleware.go new file mode 100644 index 00000000..2dd46d62 --- /dev/null +++ b/src/internal/api/middleware.go @@ -0,0 +1,149 @@ +package api + +import ( + "context" + "errors" + "log/slog" + "net/http" + "strings" + "time" + + "github.com/google/uuid" + + "mtusi-messenger/internal/apperr" + "mtusi-messenger/internal/auth" + "mtusi-messenger/internal/model" +) + +type ctxKey int + +const ( + ctxRequestID ctxKey = iota + ctxUser + ctxToken +) + +func requestIDMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + id := r.Header.Get("X-Request-Id") + if id == "" { + id = uuid.NewString() + } + w.Header().Set("X-Request-Id", id) + next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), ctxRequestID, id))) + }) +} + +func instanceIDMiddleware(instanceID string, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Instance-Id", instanceID) + next.ServeHTTP(w, r) + }) +} + +func requestID(r *http.Request) string { + if id, ok := r.Context().Value(ctxRequestID).(string); ok && id != "" { + return id + } + return "unknown" +} + +type statusWriter struct { + http.ResponseWriter + status int +} + +func (s *statusWriter) WriteHeader(code int) { + s.status = code + s.ResponseWriter.WriteHeader(code) +} + +func loggingMiddleware(log *slog.Logger, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + sw := &statusWriter{ResponseWriter: w, status: http.StatusOK} + next.ServeHTTP(sw, r) + + log.Info("http", + "method", r.Method, + "path", r.URL.Path, + "status", sw.status, + "dur_ms", time.Since(start).Milliseconds(), + "request_id", requestID(r), + ) + }) +} + +func recoverMiddleware(log *slog.Logger, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if rec := recover(); rec != nil { + log.Error("panic recovered", "path", r.URL.Path, "request_id", requestID(r)) + apperr.Write(w, requestID(r), apperr.Internal("internal error")) + } + }() + next.ServeHTTP(w, r) + }) +} + +func timeoutMiddleware(d, binary time.Duration, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + limit := d + if strings.Contains(r.URL.Path, "/attachments") { + limit = binary + } + ctx, cancel := context.WithTimeout(r.Context(), limit) + defer cancel() + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +const bearerChallenge = `Bearer realm="messenger"` + +func unauthorizedBearer(w http.ResponseWriter, r *http.Request) { + w.Header().Set("WWW-Authenticate", bearerChallenge) + apperr.Write(w, requestID(r), apperr.Unauthorized("valid Bearer session token is required")) +} + +func bearerToken(r *http.Request) (string, bool) { + scheme, token, ok := strings.Cut(r.Header.Get("Authorization"), " ") + if !ok || !strings.EqualFold(scheme, "Bearer") { + return "", false + } + token = strings.TrimSpace(token) + if token == "" || len(token) > 256 { + return "", false + } + return token, true +} + +func (s *Server) auth(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + token, ok := bearerToken(r) + if !ok { + unauthorizedBearer(w, r) + return + } + user, err := s.sessions.Resolve(r.Context(), token) + if errors.Is(err, auth.ErrInvalidSession) { + unauthorizedBearer(w, r) + return + } + if err != nil { + + s.storeError(w, r, err) + return + } + ctx := context.WithValue(r.Context(), ctxUser, user) + ctx = context.WithValue(ctx, ctxToken, token) + next.ServeHTTP(w, r.WithContext(ctx)) + } +} + +func currentUser(r *http.Request) *model.User { + return r.Context().Value(ctxUser).(*model.User) +} + +func currentToken(r *http.Request) string { + return r.Context().Value(ctxToken).(string) +} diff --git a/src/internal/api/router.go b/src/internal/api/router.go new file mode 100644 index 00000000..4907cf11 --- /dev/null +++ b/src/internal/api/router.go @@ -0,0 +1,92 @@ +package api + +import ( + "log/slog" + "net/http" + "sync/atomic" + "time" + + "golang.org/x/sync/semaphore" + + "mtusi-messenger/internal/auth" + "mtusi-messenger/internal/objstore" + "mtusi-messenger/internal/relay" + "mtusi-messenger/internal/store" +) + +type Server struct { + store *store.Store + objects *objstore.Store + sessions *auth.Sessions + hasher *auth.Hasher + log *slog.Logger + ready *atomic.Bool + timeout time.Duration + + instanceID string + idempotencyTTL time.Duration + + uploadTimeout time.Duration + s3Timeout time.Duration + maxUpload int64 + maxPixels int64 + uploadSem *semaphore.Weighted + + relay *relay.Relay + maxActiveJobs int + pipelineVersion int + maxAttempts int + faultInjection bool +} + +type Options struct { + InstanceID string + IdempotencyTTL time.Duration + + MaxUploadBytes int64 + MaxImagePixels int64 + UploadConcurrency int64 + UploadTimeout time.Duration + S3Timeout time.Duration + + Relay *relay.Relay + MaxActiveJobs int + PipelineVersion int + MaxAttempts int + FaultInjection bool +} + +func NewServer(st *store.Store, objects *objstore.Store, sessions *auth.Sessions, hasher *auth.Hasher, log *slog.Logger, ready *atomic.Bool, opt Options) *Server { + return &Server{store: st, objects: objects, sessions: sessions, hasher: hasher, log: log, ready: ready, timeout: 8 * time.Second, + instanceID: opt.InstanceID, idempotencyTTL: opt.IdempotencyTTL, + uploadTimeout: opt.UploadTimeout, s3Timeout: opt.S3Timeout, + maxUpload: opt.MaxUploadBytes, maxPixels: opt.MaxImagePixels, + uploadSem: semaphore.NewWeighted(opt.UploadConcurrency), + relay: opt.Relay, maxActiveJobs: opt.MaxActiveJobs, pipelineVersion: opt.PipelineVersion, + maxAttempts: opt.MaxAttempts, faultInjection: opt.FaultInjection} +} + +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/health/live", s.healthLive) + mux.HandleFunc("/health/ready", s.healthReady) + mux.HandleFunc("/api/v1/users", s.usersRoute) + mux.HandleFunc("/api/v1/auth/sessions", s.createSession) + mux.HandleFunc("/api/v1/auth/sessions/current", s.auth(s.revokeSession)) + mux.HandleFunc("/api/v1/users/me", s.auth(s.me)) + mux.HandleFunc("/api/v1/chats", s.auth(s.chatsRoute)) + mux.HandleFunc("/api/v1/chats/{chat_id}", s.auth(s.chatRoute)) + mux.HandleFunc("/api/v1/chats/{chat_id}/messages", s.auth(s.messagesRoute)) + mux.HandleFunc("/api/v1/chats/{chat_id}/attachments", s.auth(s.uploadAttachment)) + mux.HandleFunc("/api/v1/attachments/{attachment_id}", s.auth(s.getAttachment)) + mux.HandleFunc("/api/v1/attachments/{attachment_id}/content", s.auth(s.downloadAttachment)) + mux.HandleFunc("/", s.notFound) + + var h http.Handler = mux + h = timeoutMiddleware(s.timeout, s.uploadTimeout, h) + h = recoverMiddleware(s.log, h) + h = loggingMiddleware(s.log, h) + h = requestIDMiddleware(h) + h = instanceIDMiddleware(s.instanceID, h) + return h +} diff --git a/src/internal/apperr/errors.go b/src/internal/apperr/errors.go new file mode 100644 index 00000000..0c6d7c5e --- /dev/null +++ b/src/internal/apperr/errors.go @@ -0,0 +1,64 @@ +package apperr + +import ( + "encoding/json" + "net/http" +) + +type Code string + +const ( + CodeInvalidRequest Code = "invalid_request" + CodeUnauthorized Code = "unauthorized" + CodeNotFound Code = "not_found" + CodeConflict Code = "conflict" + CodePayloadTooLarge Code = "payload_too_large" + CodeUnsupportedMedia Code = "unsupported_media_type" + CodeRateLimited Code = "rate_limited" + CodeUnavailable Code = "unavailable" + CodeInternal Code = "internal_error" +) + +type Body struct { + Code Code `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} + +type Envelope struct { + Error Body `json:"error"` +} + +type AppError struct { + Status int + Code Code + Msg string +} + +func (e *AppError) Error() string { return e.Msg } + +func New(status int, code Code, msg string) *AppError { + return &AppError{Status: status, Code: code, Msg: msg} +} + +func InvalidRequest(msg string) *AppError { return New(http.StatusBadRequest, CodeInvalidRequest, msg) } +func Unauthorized(msg string) *AppError { return New(http.StatusUnauthorized, CodeUnauthorized, msg) } +func NotFound(msg string) *AppError { return New(http.StatusNotFound, CodeNotFound, msg) } +func Conflict(msg string) *AppError { return New(http.StatusConflict, CodeConflict, msg) } +func PayloadTooLarge(msg string) *AppError { + return New(http.StatusRequestEntityTooLarge, CodePayloadTooLarge, msg) +} +func UnsupportedMedia(msg string) *AppError { + return New(http.StatusUnsupportedMediaType, CodeUnsupportedMedia, msg) +} +func Unavailable(msg string) *AppError { + return New(http.StatusServiceUnavailable, CodeUnavailable, msg) +} +func Internal(msg string) *AppError { return New(http.StatusInternalServerError, CodeInternal, msg) } + +func Write(w http.ResponseWriter, requestID string, err *AppError) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("X-Request-Id", requestID) + w.WriteHeader(err.Status) + _ = json.NewEncoder(w).Encode(Envelope{Error: Body{Code: err.Code, Message: err.Msg, RequestID: requestID}}) +} diff --git a/src/internal/auth/password.go b/src/internal/auth/password.go new file mode 100644 index 00000000..ceaf8fe4 --- /dev/null +++ b/src/internal/auth/password.go @@ -0,0 +1,100 @@ +package auth + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "errors" + "fmt" + "strings" + + "golang.org/x/crypto/argon2" +) + +const ( + argonMemoryKiB = 19 * 1024 + argonTime = 2 + argonThreads = 1 + argonKeyLen = 32 + argonSaltLen = 16 +) + +var ErrBadHash = errors.New("malformed password hash") + +type Hasher struct { + slots chan struct{} + dummy string +} + +func NewHasher(concurrency int) *Hasher { + h := &Hasher{slots: make(chan struct{}, concurrency)} + + salt := make([]byte, argonSaltLen) + _, _ = rand.Read(salt) + h.dummy = encode(salt, argon2.IDKey([]byte("dummy-password-x"), salt, argonTime, argonMemoryKiB, argonThreads, argonKeyLen)) + return h +} + +func (h *Hasher) acquire(ctx context.Context) error { + select { + case h.slots <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (h *Hasher) release() { <-h.slots } + +func encode(salt, key []byte) string { + return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", argon2.Version, argonMemoryKiB, argonTime, argonThreads, + base64.RawStdEncoding.EncodeToString(salt), base64.RawStdEncoding.EncodeToString(key)) +} + +func (h *Hasher) Hash(ctx context.Context, password string) (string, error) { + if err := h.acquire(ctx); err != nil { + return "", err + } + defer h.release() + salt := make([]byte, argonSaltLen) + if _, err := rand.Read(salt); err != nil { + return "", err + } + return encode(salt, argon2.IDKey([]byte(password), salt, argonTime, argonMemoryKiB, argonThreads, argonKeyLen)), nil +} + +func (h *Hasher) Verify(ctx context.Context, password, encoded string) (bool, error) { + missing := encoded == "" + if missing { + encoded = h.dummy + } + parts := strings.Split(encoded, "$") + if len(parts) != 6 || parts[1] != "argon2id" { + return false, ErrBadHash + } + var version int + var mem uint32 + var t uint32 + var p uint8 + if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil || version != argon2.Version { + return false, ErrBadHash + } + if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &mem, &t, &p); err != nil { + return false, ErrBadHash + } + salt, err := base64.RawStdEncoding.DecodeString(parts[4]) + if err != nil { + return false, ErrBadHash + } + want, err := base64.RawStdEncoding.DecodeString(parts[5]) + if err != nil { + return false, ErrBadHash + } + if err := h.acquire(ctx); err != nil { + return false, err + } + got := argon2.IDKey([]byte(password), salt, t, mem, p, uint32(len(want))) + h.release() + return subtle.ConstantTimeCompare(got, want) == 1 && !missing, nil +} diff --git a/src/internal/auth/sessions.go b/src/internal/auth/sessions.go new file mode 100644 index 00000000..cf0a51ec --- /dev/null +++ b/src/internal/auth/sessions.go @@ -0,0 +1,137 @@ +package auth + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "errors" + "log/slog" + "time" + + "mtusi-messenger/internal/model" + "mtusi-messenger/internal/store" +) + +var ( + ErrInvalidSession = errors.New("invalid session") + ErrCacheUnavailable = errors.New("session cache unavailable") +) + +type SessionCache interface { + Get(ctx context.Context, key string) (*CachedSession, error) + Set(ctx context.Context, key string, s *CachedSession, ttl time.Duration) error + Del(ctx context.Context, key string) error +} + +type CachedSession struct { + User model.User `json:"user"` + ExpiresAt time.Time `json:"exp"` +} + +type Sessions struct { + store *store.Store + cache SessionCache + ttl time.Duration + cacheMaxTTL time.Duration + log *slog.Logger +} + +func NewSessions(st *store.Store, cache SessionCache, ttl, cacheMaxTTL time.Duration, log *slog.Logger) *Sessions { + return &Sessions{store: st, cache: cache, ttl: ttl, cacheMaxTTL: cacheMaxTTL, log: log} +} + +func NewToken() (string, []byte, error) { + raw := make([]byte, 32) + if _, err := rand.Read(raw); err != nil { + return "", nil, err + } + token := base64.RawURLEncoding.EncodeToString(raw) + return token, HashToken(token), nil +} + +func HashToken(token string) []byte { + sum := sha256.Sum256([]byte(token)) + return sum[:] +} + +func cacheKey(hash []byte) string { + return "sess:" + base64.RawURLEncoding.EncodeToString(hash) +} + +func (s *Sessions) Create(ctx context.Context, user *model.User) (string, time.Time, error) { + token, hash, err := NewToken() + if err != nil { + return "", time.Time{}, err + } + expires, err := s.store.CreateSession(ctx, hash, user.ID, s.ttl) + if err != nil { + return "", time.Time{}, err + } + s.cacheSet(ctx, hash, user, expires) + return token, expires, nil +} + +func (s *Sessions) Resolve(ctx context.Context, token string) (*model.User, error) { + hash := HashToken(token) + if s.cache != nil { + cs, err := s.cache.Get(ctx, cacheKey(hash)) + if err == nil && cs != nil { + if time.Now().Before(cs.ExpiresAt) { + u := cs.User + return &u, nil + } + + } else if err != nil { + s.log.Debug("session cache get failed, falling back to postgres", "error", err.Error()) + } + } + user, expires, err := s.store.SessionUser(ctx, hash) + if errors.Is(err, store.ErrNotFound) { + return nil, ErrInvalidSession + } + if err != nil { + return nil, err + } + s.cacheSet(ctx, hash, user, expires) + return user, nil +} + +func (s *Sessions) Revoke(ctx context.Context, token string) error { + hash := HashToken(token) + if s.cache != nil { + if err := s.cache.Del(ctx, cacheKey(hash)); err != nil { + return ErrCacheUnavailable + } + } + err := s.store.RevokeSession(ctx, hash) + if errors.Is(err, store.ErrNotFound) { + return ErrInvalidSession + } + if err != nil { + return err + } + if s.cache != nil { + if err := s.cache.Del(ctx, cacheKey(hash)); err != nil { + + s.log.Warn("session cache delete after revoke failed", "error", err.Error()) + } + } + return nil +} + +func (s *Sessions) cacheSet(ctx context.Context, hash []byte, user *model.User, expires time.Time) { + if s.cache == nil { + return + } + ttl := time.Until(expires) + if ttl > s.cacheMaxTTL { + ttl = s.cacheMaxTTL + } + if ttl <= 0 { + return + } + if err := s.cache.Set(ctx, cacheKey(hash), &CachedSession{User: *user, ExpiresAt: expires}, ttl); err != nil { + s.log.Debug("session cache set failed", "error", err.Error()) + } +} diff --git a/src/internal/broker/rabbit.go b/src/internal/broker/rabbit.go new file mode 100644 index 00000000..ee6bd61f --- /dev/null +++ b/src/internal/broker/rabbit.go @@ -0,0 +1,125 @@ +package broker + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + amqp "github.com/rabbitmq/amqp091-go" +) + +const ( + Exchange = "attachments" + RoutingKey = "attachment.process.v1" + Queue = "attachment.process.v1" + DLX = "attachments.dlx" + DLQ = "attachment.process.dlq" +) + +func Declare(ch *amqp.Channel) error { + if err := ch.ExchangeDeclare(Exchange, "direct", true, false, false, false, nil); err != nil { + return err + } + if err := ch.ExchangeDeclare(DLX, "fanout", true, false, false, false, nil); err != nil { + return err + } + if _, err := ch.QueueDeclare(DLQ, true, false, false, false, amqp.Table{"x-queue-type": "quorum"}); err != nil { + return err + } + if err := ch.QueueBind(DLQ, "", DLX, false, nil); err != nil { + return err + } + if _, err := ch.QueueDeclare(Queue, true, false, false, false, amqp.Table{ + "x-queue-type": "quorum", + "x-dead-letter-exchange": DLX, + "x-delivery-limit": int32(20), + }); err != nil { + return err + } + return ch.QueueBind(Queue, RoutingKey, Exchange, false, nil) +} + +func Dial(url string) (*amqp.Connection, error) { + return amqp.DialConfig(url, amqp.Config{Heartbeat: 10 * time.Second, Locale: "en_US", + Properties: amqp.Table{"connection_name": "messenger"}}) +} + +type Publisher struct { + url string + mu sync.Mutex + conn *amqp.Connection + ch *amqp.Channel +} + +func NewPublisher(url string) *Publisher { return &Publisher{url: url} } + +func (p *Publisher) channel() (*amqp.Channel, error) { + if p.ch != nil && !p.ch.IsClosed() && p.conn != nil && !p.conn.IsClosed() { + return p.ch, nil + } + p.closeLocked() + conn, err := Dial(p.url) + if err != nil { + return nil, err + } + ch, err := conn.Channel() + if err != nil { + conn.Close() + return nil, err + } + if err := ch.Confirm(false); err != nil { + conn.Close() + return nil, err + } + if err := Declare(ch); err != nil { + conn.Close() + return nil, err + } + p.conn, p.ch = conn, ch + return ch, nil +} + +func (p *Publisher) closeLocked() { + if p.conn != nil { + _ = p.conn.Close() + } + p.conn, p.ch = nil, nil +} + +func (p *Publisher) Publish(ctx context.Context, messageID string, body []byte) error { + p.mu.Lock() + defer p.mu.Unlock() + ch, err := p.channel() + if err != nil { + return fmt.Errorf("broker connect: %w", err) + } + conf, err := ch.PublishWithDeferredConfirmWithContext(ctx, Exchange, RoutingKey, false, false, amqp.Publishing{ + ContentType: "application/json", + DeliveryMode: amqp.Persistent, + MessageId: messageID, + Type: "attachment.process.v1", + Timestamp: time.Now().UTC(), + Body: body, + }) + if err != nil { + p.closeLocked() + return fmt.Errorf("broker publish: %w", err) + } + ok, err := conf.WaitContext(ctx) + if err != nil { + p.closeLocked() + return fmt.Errorf("broker confirm: %w", err) + } + if !ok { + return errors.New("broker nacked publish") + } + return nil +} + +func (p *Publisher) Close() { + p.mu.Lock() + defer p.mu.Unlock() + p.closeLocked() +} diff --git a/src/internal/cache/valkey.go b/src/internal/cache/valkey.go new file mode 100644 index 00000000..591d9e49 --- /dev/null +++ b/src/internal/cache/valkey.go @@ -0,0 +1,80 @@ +package cache + +import ( + "context" + "encoding/json" + "errors" + "sync/atomic" + "time" + + "github.com/redis/go-redis/v9" + + "mtusi-messenger/internal/auth" +) + +var ErrOpen = errors.New("cache circuit open") + +type Valkey struct { + rdb *redis.Client + openUntil atomic.Int64 + cooldown time.Duration +} + +func NewValkey(url string) (*Valkey, error) { + opt, err := redis.ParseURL(url) + if err != nil { + return nil, err + } + opt.DialTimeout = 500 * time.Millisecond + opt.ReadTimeout = 300 * time.Millisecond + opt.WriteTimeout = 300 * time.Millisecond + opt.PoolSize = 10 + opt.MaxRetries = 0 + return &Valkey{rdb: redis.NewClient(opt), cooldown: 2 * time.Second}, nil +} + +func (v *Valkey) Close() error { return v.rdb.Close() } + +func (v *Valkey) Ping(ctx context.Context) error { return v.guard(v.rdb.Ping(ctx).Err()) } + +func (v *Valkey) open() bool { return time.Now().UnixNano() < v.openUntil.Load() } + +func (v *Valkey) guard(err error) error { + if err != nil && !errors.Is(err, redis.Nil) { + v.openUntil.Store(time.Now().Add(v.cooldown).UnixNano()) + } + return err +} + +func (v *Valkey) Get(ctx context.Context, key string) (*auth.CachedSession, error) { + if v.open() { + return nil, ErrOpen + } + raw, err := v.rdb.Get(ctx, key).Bytes() + if errors.Is(err, redis.Nil) { + return nil, nil + } + if err := v.guard(err); err != nil { + return nil, err + } + var cs auth.CachedSession + if err := json.Unmarshal(raw, &cs); err != nil { + return nil, nil + } + return &cs, nil +} + +func (v *Valkey) Set(ctx context.Context, key string, s *auth.CachedSession, ttl time.Duration) error { + if v.open() { + return ErrOpen + } + raw, err := json.Marshal(s) + if err != nil { + return err + } + return v.guard(v.rdb.Set(ctx, key, raw, ttl).Err()) +} + +func (v *Valkey) Del(ctx context.Context, key string) error { + return v.guard(v.rdb.Del(ctx, key).Err()) +} diff --git a/src/internal/db/migrate.go b/src/internal/db/migrate.go new file mode 100644 index 00000000..96d0c779 --- /dev/null +++ b/src/internal/db/migrate.go @@ -0,0 +1,122 @@ +package db + +import ( + "context" + "embed" + "fmt" + "io/fs" + "log/slog" + "sort" + "strconv" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" +) + +//go:embed migrations/*.sql +var migrationFS embed.FS + +const advisoryLockID = 20260202 + +type migration struct { + version int + name string + sql string +} + +func loadMigrations() ([]migration, error) { + entries, err := fs.ReadDir(migrationFS, "migrations") + if err != nil { + return nil, err + } + var out []migration + for _, e := range entries { + name := e.Name() + prefix, _, ok := strings.Cut(name, "_") + if !ok || !strings.HasSuffix(name, ".sql") { + return nil, fmt.Errorf("bad migration file name %q", name) + } + v, err := strconv.Atoi(prefix) + if err != nil { + return nil, fmt.Errorf("bad migration version in %q", name) + } + body, err := migrationFS.ReadFile("migrations/" + name) + if err != nil { + return nil, err + } + out = append(out, migration{version: v, name: name, sql: string(body)}) + } + sort.Slice(out, func(i, j int) bool { return out[i].version < out[j].version }) + for i := 1; i < len(out); i++ { + if out[i].version == out[i-1].version { + return nil, fmt.Errorf("duplicate migration version %d", out[i].version) + } + } + return out, nil +} + +func Migrate(ctx context.Context, pool *pgxpool.Pool, log *slog.Logger) error { + migs, err := loadMigrations() + if err != nil { + return err + } + conn, err := pool.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + + if _, err := conn.Exec(ctx, "SELECT pg_advisory_lock($1)", advisoryLockID); err != nil { + return err + } + defer conn.Exec(context.Background(), "SELECT pg_advisory_unlock($1)", advisoryLockID) + + if _, err := conn.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations ( + version integer PRIMARY KEY, + name text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now())`); err != nil { + return err + } + + applied := map[int]bool{} + rows, err := conn.Query(ctx, "SELECT version FROM schema_migrations") + if err != nil { + return err + } + for rows.Next() { + var v int + if err := rows.Scan(&v); err != nil { + rows.Close() + return err + } + applied[v] = true + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + + for _, m := range migs { + if applied[m.version] { + continue + } + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + if _, err := tx.Exec(ctx, m.sql); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("migration %s: %w", m.name, err) + } + if _, err := tx.Exec(ctx, "INSERT INTO schema_migrations (version, name) VALUES ($1, $2)", m.version, m.name); err != nil { + _ = tx.Rollback(ctx) + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + log.Info("migration applied", "version", m.version, "name", m.name) + } + log.Info("migrations up to date", "total", len(migs)) + return nil +} diff --git a/src/internal/db/migrations/0001_init.sql b/src/internal/db/migrations/0001_init.sql new file mode 100644 index 00000000..a500b303 --- /dev/null +++ b/src/internal/db/migrations/0001_init.sql @@ -0,0 +1,51 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + CONSTRAINT users_username_key UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_by uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id), + position smallint NOT NULL, + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_position_key UNIQUE (chat_id, position) +); +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000), + CONSTRAINT messages_sender_member_fk FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE +); +CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32), + CONSTRAINT sessions_expiry_after_create CHECK (expires_at > created_at) +); +CREATE INDEX sessions_user_idx ON sessions (user_id); +CREATE INDEX sessions_expires_idx ON sessions (expires_at); diff --git a/src/internal/db/migrations/0002_message_idempotency.sql b/src/internal/db/migrations/0002_message_idempotency.sql new file mode 100644 index 00000000..5103b78d --- /dev/null +++ b/src/internal/db/migrations/0002_message_idempotency.sql @@ -0,0 +1,13 @@ +CREATE TABLE message_idempotency ( + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + idem_key text NOT NULL, + request_hash bytea NOT NULL, + message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, chat_id, idem_key), + CONSTRAINT message_idempotency_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT message_idempotency_hash_len CHECK (octet_length(request_hash) = 32) +); +CREATE INDEX message_idempotency_created_idx ON message_idempotency (created_at); diff --git a/src/internal/db/migrations/0003_attachments.sql b/src/internal/db/migrations/0003_attachments.sql new file mode 100644 index 00000000..364ef2a0 --- /dev/null +++ b/src/internal/db/migrations/0003_attachments.sql @@ -0,0 +1,44 @@ +ALTER TABLE messages DROP CONSTRAINT messages_text_len; +ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 0 AND 4000); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + owner_id uuid NOT NULL, + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + object_key text NOT NULL, + status text NOT NULL, + missing_at timestamptz, + created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + updated_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + CONSTRAINT attachments_object_key_key UNIQUE (object_key), + CONSTRAINT attachments_chat_id_key UNIQUE (chat_id, id), + CONSTRAINT attachments_owner_member_fk FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE, + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_pixels CHECK (width > 0 AND height > 0 AND width::bigint * height <= 40000000), + CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready')) +); +CREATE INDEX attachments_pending_idx ON attachments (created_at) WHERE status = 'pending'; + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED, + chat_id uuid NOT NULL, + attachment_id uuid NOT NULL, + position smallint NOT NULL, + PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position_key UNIQUE (message_id, position), + CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9), + CONSTRAINT message_attachments_same_chat_fk FOREIGN KEY (chat_id, attachment_id) + REFERENCES attachments (chat_id, id) +); +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/src/internal/db/migrations/0004_jobs.sql b/src/internal/db/migrations/0004_jobs.sql new file mode 100644 index 00000000..3808ce90 --- /dev/null +++ b/src/internal/db/migrations/0004_jobs.sql @@ -0,0 +1,75 @@ +ALTER TABLE attachments DROP CONSTRAINT attachments_status; +ALTER TABLE attachments ADD CONSTRAINT attachments_status + CHECK (status IN ('pending', 'queued', 'processing', 'ready', 'failed')); +ALTER TABLE attachments ADD COLUMN staging_key text; +ALTER TABLE attachments ADD COLUMN error_code text; +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1; +ALTER TABLE attachments ADD CONSTRAINT attachments_error_code + CHECK ((status = 'failed') = (error_code IS NOT NULL)); + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + name text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL, + created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + PRIMARY KEY (attachment_id, pipeline_version, name), + CONSTRAINT attachment_variants_object_key_key UNIQUE (object_key), + CONSTRAINT attachment_variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')), + CONSTRAINT attachment_variants_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachment_variants_dims CHECK (width > 0 AND height > 0 AND size_bytes > 0) +); + +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + status text NOT NULL, + attempt integer NOT NULL DEFAULT 1, + max_attempts integer NOT NULL, + next_attempt_at timestamptz, + lease_owner text, + lease_until timestamptz, + error_code text, + last_error text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + started_at timestamptz, + finished_at timestamptz, + CONSTRAINT attachment_jobs_effect_key UNIQUE (attachment_id, pipeline_version), + CONSTRAINT attachment_jobs_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + CONSTRAINT attachment_jobs_attempt CHECK (attempt >= 1 AND max_attempts >= 1) +); +CREATE INDEX attachment_jobs_active_idx ON attachment_jobs (status, lease_until) + WHERE status IN ('queued', 'processing'); +CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (finished_at) WHERE status = 'failed'; + +CREATE TABLE job_outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id) ON DELETE CASCADE, + attempt integer NOT NULL, + payload jsonb NOT NULL, + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + published_at timestamptz, + publish_attempts integer NOT NULL DEFAULT 0, + last_error text +); +CREATE INDEX job_outbox_unpublished_idx ON job_outbox (available_at, id) WHERE published_at IS NULL; + +CREATE TABLE job_events ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id) ON DELETE CASCADE, + attempt integer NOT NULL, + event text NOT NULL, + error_code text, + detail text, + actor text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX job_events_job_idx ON job_events (job_id, id); diff --git a/src/internal/db/pool.go b/src/internal/db/pool.go new file mode 100644 index 00000000..3c818a19 --- /dev/null +++ b/src/internal/db/pool.go @@ -0,0 +1,31 @@ +package db + +import ( + "context" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +type PoolOptions struct { + MaxConns int32 +} + +func Open(ctx context.Context, url string, opt PoolOptions) (*pgxpool.Pool, error) { + cfg, err := pgxpool.ParseConfig(url) + if err != nil { + return nil, err + } + cfg.MaxConns = opt.MaxConns + cfg.MinConns = 1 + cfg.MaxConnLifetime = 30 * time.Minute + cfg.MaxConnIdleTime = 5 * time.Minute + cfg.HealthCheckPeriod = 15 * time.Second + cfg.ConnConfig.ConnectTimeout = 3 * time.Second + rp := cfg.ConnConfig.RuntimeParams + rp["application_name"] = "mtusi-messenger" + rp["statement_timeout"] = "5000" + rp["lock_timeout"] = "2000" + rp["idle_in_transaction_session_timeout"] = "10000" + return pgxpool.NewWithConfig(ctx, cfg) +} diff --git a/src/internal/imaging/imaging.go b/src/internal/imaging/imaging.go new file mode 100644 index 00000000..8a5db206 --- /dev/null +++ b/src/internal/imaging/imaging.go @@ -0,0 +1,289 @@ +package imaging + +import ( + "bytes" + "context" + "errors" + "fmt" + "image" + "image/color" + "image/jpeg" + "image/png" + + "github.com/gen2brain/webp" + "golang.org/x/image/draw" + "golang.org/x/sync/semaphore" +) + +const PipelineVersion = 1 + +const ( + ThumbBox = 256 + CropSide = 128 + OptimizedBox = 1280 + JPEGQuality = 85 + WebPQuality = 80 + WebPMethod = 4 +) + +type PermanentError struct{ Code, Detail string } + +func (e *PermanentError) Error() string { return e.Code + ": " + e.Detail } + +func permanent(code, format string, a ...any) error { + return &PermanentError{Code: code, Detail: fmt.Sprintf(format, a...)} +} + +type Output struct { + Name string + ContentType string + Ext string + Data []byte + Width int + Height int +} + +type Limits struct { + MaxBytes int64 + MaxPixels int64 + + Budget *semaphore.Weighted + + BudgetSize int64 +} + +func Fit(w, h, box int) (int, int) { + if w <= box && h <= box { + return w, h + } + if w >= h { + return box, max(1, h*box/w) + } + return max(1, w*box/h), box +} + +func Cost(w, h int) int64 { + px := int64(w) * int64(h) + return px*4*2 + OptimizedBox*OptimizedBox*4 +} + +func Process(ctx context.Context, data []byte, contentType string, lim Limits) ([]Output, error) { + if int64(len(data)) > lim.MaxBytes { + return nil, permanent("image_too_large", "original exceeds %d bytes", lim.MaxBytes) + } + cfg, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || cfg.Width <= 0 || cfg.Height <= 0 { + return nil, permanent("invalid_image", "image header is invalid") + } + if (format == "png") != (contentType == "image/png") || (format == "jpeg") != (contentType == "image/jpeg") { + return nil, permanent("invalid_image", "format does not match content type") + } + if int64(cfg.Width)*int64(cfg.Height) > lim.MaxPixels { + return nil, permanent("image_too_large", "image exceeds %d pixels", lim.MaxPixels) + } + cost := min(Cost(cfg.Width, cfg.Height), lim.BudgetSize) + if err := lim.Budget.Acquire(ctx, cost); err != nil { + return nil, err + } + type result struct { + out []Output + err error + } + done := make(chan result, 1) + go func() { + defer lim.Budget.Release(cost) + defer func() { + if p := recover(); p != nil { + done <- result{err: permanent("invalid_image", "decoder panic")} + } + }() + out, err := build(data, format) + done <- result{out, err} + }() + select { + case r := <-done: + return r.out, r.err + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func build(data []byte, format string) ([]Output, error) { + var src image.Image + var err error + orientation := 1 + if format == "png" { + src, err = png.Decode(bytes.NewReader(data)) + } else { + src, err = jpeg.Decode(bytes.NewReader(data)) + orientation = jpegOrientation(data) + } + if err != nil { + return nil, permanent("invalid_image", "image data is corrupted") + } + img := flatten(orient(src, orientation)) + b := img.Bounds() + w, h := b.Dx(), b.Dy() + + var outs []Output + tw, th := Fit(w, h, ThumbBox) + thumb, err := encodeJPEG(resize(img, b, tw, th)) + if err != nil { + return nil, err + } + outs = append(outs, Output{Name: "thumbnail", ContentType: "image/jpeg", Ext: "jpg", Data: thumb, Width: tw, Height: th}) + + side := min(w, h) + sq := image.Rect(b.Min.X+(w-side)/2, b.Min.Y+(h-side)/2, b.Min.X+(w-side)/2+side, b.Min.Y+(h-side)/2+side) + crop, err := encodeJPEG(resize(img, sq, CropSide, CropSide)) + if err != nil { + return nil, err + } + outs = append(outs, Output{Name: "crop", ContentType: "image/jpeg", Ext: "jpg", Data: crop, Width: CropSide, Height: CropSide}) + + ow, oh := Fit(w, h, OptimizedBox) + var opt image.Image = img + if ow != w || oh != h { + opt = resize(img, b, ow, oh) + } + var buf bytes.Buffer + if err := webp.Encode(&buf, opt, webp.Options{Quality: WebPQuality, Method: WebPMethod}); err != nil { + return nil, fmt.Errorf("webp encode: %w", err) + } + outs = append(outs, Output{Name: "optimized", ContentType: "image/webp", Ext: "webp", Data: buf.Bytes(), Width: ow, Height: oh}) + return outs, nil +} + +func resize(src image.Image, sr image.Rectangle, w, h int) *image.RGBA { + dst := image.NewRGBA(image.Rect(0, 0, w, h)) + draw.CatmullRom.Scale(dst, dst.Bounds(), src, sr, draw.Src, nil) + return dst +} + +func encodeJPEG(img image.Image) ([]byte, error) { + var buf bytes.Buffer + if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: JPEGQuality}); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func flatten(src image.Image) image.Image { + if op, ok := src.(interface{ Opaque() bool }); ok && op.Opaque() { + return src + } + b := src.Bounds() + dst := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy())) + draw.Draw(dst, dst.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src) + draw.Draw(dst, dst.Bounds(), src, b.Min, draw.Over) + return dst +} + +func orient(src image.Image, o int) image.Image { + if o < 2 || o > 8 { + return src + } + b := src.Bounds() + w, h := b.Dx(), b.Dy() + dw, dh := w, h + if o >= 5 { + dw, dh = h, w + } + dst := image.NewRGBA(image.Rect(0, 0, dw, dh)) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + var dx, dy int + switch o { + case 2: + dx, dy = w-1-x, y + case 3: + dx, dy = w-1-x, h-1-y + case 4: + dx, dy = x, h-1-y + case 5: + dx, dy = y, x + case 6: + dx, dy = h-1-y, x + case 7: + dx, dy = h-1-y, w-1-x + case 8: + dx, dy = y, w-1-x + } + dst.Set(dx, dy, src.At(b.Min.X+x, b.Min.Y+y)) + } + } + return dst +} + +var errNoExif = errors.New("no exif") + +func jpegOrientation(data []byte) int { + o, err := parseOrientation(data) + if err != nil { + return 1 + } + return o +} + +func parseOrientation(d []byte) (int, error) { + if len(d) < 4 || d[0] != 0xff || d[1] != 0xd8 { + return 0, errNoExif + } + i := 2 + for i+4 <= len(d) { + if d[i] != 0xff { + return 0, errNoExif + } + marker := d[i+1] + if marker == 0xda || marker == 0xd9 { + return 0, errNoExif + } + l := int(d[i+2])<<8 | int(d[i+3]) + if l < 2 || i+2+l > len(d) { + return 0, errNoExif + } + seg := d[i+4 : i+2+l] + if marker == 0xe1 && len(seg) > 14 && string(seg[:6]) == "Exif\x00\x00" { + return tiffOrientation(seg[6:]) + } + i += 2 + l + } + return 0, errNoExif +} + +func tiffOrientation(t []byte) (int, error) { + if len(t) < 8 { + return 0, errNoExif + } + var u16 func([]byte) int + var u32 func([]byte) int + switch string(t[:2]) { + case "II": + u16 = func(b []byte) int { return int(b[0]) | int(b[1])<<8 } + u32 = func(b []byte) int { return int(b[0]) | int(b[1])<<8 | int(b[2])<<16 | int(b[3])<<24 } + case "MM": + u16 = func(b []byte) int { return int(b[1]) | int(b[0])<<8 } + u32 = func(b []byte) int { return int(b[3]) | int(b[2])<<8 | int(b[1])<<16 | int(b[0])<<24 } + default: + return 0, errNoExif + } + off := u32(t[4:8]) + if off < 8 || off+2 > len(t) { + return 0, errNoExif + } + n := u16(t[off : off+2]) + for k := 0; k < n; k++ { + e := off + 2 + k*12 + if e+12 > len(t) { + break + } + if u16(t[e:e+2]) == 0x0112 { + v := u16(t[e+8 : e+10]) + if v >= 1 && v <= 8 { + return v, nil + } + return 0, errNoExif + } + } + return 0, errNoExif +} diff --git a/src/internal/imaging/imaging_test.go b/src/internal/imaging/imaging_test.go new file mode 100644 index 00000000..84153aff --- /dev/null +++ b/src/internal/imaging/imaging_test.go @@ -0,0 +1,106 @@ +package imaging + +import ( + "bytes" + "context" + "image" + "image/color" + "image/jpeg" + "image/png" + "testing" + + "golang.org/x/sync/semaphore" +) + +func lim() Limits { + return Limits{MaxBytes: 10 << 20, MaxPixels: 40_000_000, Budget: semaphore.NewWeighted(256 << 20), BudgetSize: 256 << 20} +} + +func TestFit(t *testing.T) { + cases := []struct{ w, h, box, ew, eh int }{ + {320, 200, 256, 256, 160}, + {200, 320, 256, 160, 256}, + {100, 50, 256, 100, 50}, + {5000, 1, 256, 256, 1}, + {1, 5000, 1280, 1, 1280}, + } + for _, c := range cases { + if w, h := Fit(c.w, c.h, c.box); w != c.ew || h != c.eh { + t.Errorf("Fit(%d,%d,%d)=%d,%d want %d,%d", c.w, c.h, c.box, w, h, c.ew, c.eh) + } + } +} + +func TestProcessPNGWithAlpha(t *testing.T) { + src := image.NewNRGBA(image.Rect(0, 0, 320, 200)) + var buf bytes.Buffer + _ = png.Encode(&buf, src) + outs, err := Process(context.Background(), buf.Bytes(), "image/png", lim()) + if err != nil { + t.Fatal(err) + } + want := map[string][2]int{"thumbnail": {256, 160}, "crop": {128, 128}, "optimized": {320, 200}} + for _, o := range outs { + if d := want[o.Name]; d != [2]int{o.Width, o.Height} { + t.Errorf("%s: %dx%d want %v", o.Name, o.Width, o.Height, d) + } + if o.Name == "thumbnail" { + img, err := jpeg.Decode(bytes.NewReader(o.Data)) + if err != nil { + t.Fatal(err) + } + if img.Bounds().Dx() != 256 || img.Bounds().Dy() != 160 { + t.Fatalf("thumbnail real size %v", img.Bounds()) + } + r, g, b, _ := img.At(10, 10).RGBA() + if r>>8 < 245 || g>>8 < 245 || b>>8 < 245 { + t.Errorf("alpha not composited on white: %d %d %d", r>>8, g>>8, b>>8) + } + } + } + if len(outs) != 3 { + t.Fatalf("got %d variants", len(outs)) + } +} + +func TestExifOrientation(t *testing.T) { + src := image.NewRGBA(image.Rect(0, 0, 320, 200)) + for y := 0; y < 200; y++ { + for x := 0; x < 320; x++ { + src.Set(x, y, color.RGBA{uint8(x), uint8(y), 0, 255}) + } + } + var buf bytes.Buffer + _ = jpeg.Encode(&buf, src, nil) + data := buf.Bytes() + tiff := []byte{'M', 'M', 0, 42, 0, 0, 0, 8, 0, 1, 0x01, 0x12, 0, 3, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0} + seg := append([]byte("Exif\x00\x00"), tiff...) + app1 := append([]byte{0xff, 0xe1, byte((len(seg) + 2) >> 8), byte(len(seg) + 2)}, seg...) + withExif := append(append([]byte{0xff, 0xd8}, app1...), data[2:]...) + if o := jpegOrientation(withExif); o != 6 { + t.Fatalf("orientation %d", o) + } + outs, err := Process(context.Background(), withExif, "image/jpeg", lim()) + if err != nil { + t.Fatal(err) + } + if outs[0].Width != 160 || outs[0].Height != 256 { + t.Fatalf("thumbnail %dx%d, want 160x256", outs[0].Width, outs[0].Height) + } +} + +func TestRejects(t *testing.T) { + if _, err := Process(context.Background(), []byte("\x89PNG\r\n\x1a\nnot really"), "image/png", lim()); err == nil { + t.Fatal("corrupt PNG accepted") + } else if pe, ok := err.(*PermanentError); !ok || pe.Code != "invalid_image" { + t.Fatalf("unexpected error %v", err) + } + l := lim() + l.MaxPixels = 100 + src := image.NewRGBA(image.Rect(0, 0, 20, 20)) + var buf bytes.Buffer + _ = png.Encode(&buf, src) + if _, err := Process(context.Background(), buf.Bytes(), "image/png", l); err == nil { + t.Fatal("pixel limit not enforced") + } +} diff --git a/src/internal/model/models.go b/src/internal/model/models.go new file mode 100644 index 00000000..1bd1b784 --- /dev/null +++ b/src/internal/model/models.go @@ -0,0 +1,93 @@ +package model + +import "time" + +type Time struct { + time.Time +} + +func NewTime(t time.Time) Time { return Time{Time: t} } + +func (t Time) MarshalJSON() ([]byte, error) { + return []byte(`"` + t.UTC().Format("2006-01-02T15:04:05.000Z") + `"`), nil +} + +type User struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt Time `json:"created_at"` +} + +type Chat struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt Time `json:"created_at"` +} + +type Message struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt Time `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type Variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + Width int `json:"width"` + Height int `json:"height"` + SHA256 string `json:"sha256"` + + ObjectKey string `json:"-"` +} + +type Attachment struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt Time `json:"created_at"` + UpdatedAt Time `json:"updated_at"` + + ObjectKey string `json:"-"` + PipelineVersion int `json:"-"` +} + +type JobMessage struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt string `json:"created_at"` +} + +const JobType = "attachment.process.v1" + +type Page[T any] struct { + Items []T `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type Health struct { + Status string `json:"status"` +} + +type Session struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt Time `json:"expires_at"` + User User `json:"user"` +} diff --git a/src/internal/objstore/s3.go b/src/internal/objstore/s3.go new file mode 100644 index 00000000..6ffad335 --- /dev/null +++ b/src/internal/objstore/s3.go @@ -0,0 +1,162 @@ +package objstore + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type Config struct { + Endpoint string + Region string + Bucket string + AccessKey string + SecretKey string +} + +type Store struct { + c *minio.Client + bucket string +} + +type Object struct { + Key string + Size int64 + LastModified time.Time +} + +func New(cfg Config) (*Store, error) { + u, err := url.Parse(cfg.Endpoint) + if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") { + return nil, errors.New("S3_ENDPOINT must be http(s)://host:port") + } + if cfg.Bucket == "" || cfg.AccessKey == "" || cfg.SecretKey == "" { + return nil, errors.New("S3 bucket and credentials are required") + } + + minio.MaxRetry = 1 + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{Timeout: 2 * time.Second, KeepAlive: 30 * time.Second}).DialContext, + MaxIdleConnsPerHost: 32, + IdleConnTimeout: 90 * time.Second, + ResponseHeaderTimeout: 10 * time.Second, + } + c, err := minio.New(u.Host, &minio.Options{ + Transport: transport, + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), + Secure: u.Scheme == "https", + Region: cfg.Region, + BucketLookup: minio.BucketLookupPath, + }) + if err != nil { + return nil, err + } + return &Store{c: c, bucket: cfg.Bucket}, nil +} + +func (s *Store) Bucket() string { return s.bucket } + +func (s *Store) Ping(ctx context.Context) error { + ok, err := s.c.BucketExists(ctx, s.bucket) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("bucket %q does not exist", s.bucket) + } + return nil +} + +func (s *Store) EnsureBucket(ctx context.Context, region string) (bool, error) { + ok, err := s.c.BucketExists(ctx, s.bucket) + if err != nil || ok { + return false, err + } + if err := s.c.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: region}); err != nil { + if code := minio.ToErrorResponse(err).Code; code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return false, nil + } + return false, err + } + return true, nil +} + +func (s *Store) Put(ctx context.Context, key string, data []byte, contentType string) error { + _, err := s.c.PutObject(ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: contentType, SendContentMd5: true, DisableMultipart: true}) + return err +} + +func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) { + obj, err := s.c.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, 0, err + } + info, err := obj.Stat() + if err != nil { + obj.Close() + return nil, 0, err + } + return obj, info.Size, nil +} + +func (s *Store) Stat(ctx context.Context, key string) (Object, error) { + info, err := s.c.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}) + if err != nil { + return Object{}, err + } + return Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil +} + +func (s *Store) Remove(ctx context.Context, key string) error { + return s.c.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}) +} + +func (s *Store) List(ctx context.Context, prefix string, fn func(Object) error) error { + for info := range s.c.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) { + if info.Err != nil { + return info.Err + } + if err := fn(Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}); err != nil { + return err + } + } + return ctx.Err() +} + +func IsNotFound(err error) bool { + if err == nil { + return false + } + switch minio.ToErrorResponse(err).Code { + case "NoSuchKey", "NotFound": + return true + } + return false +} + +func (s *Store) ReadAll(ctx context.Context, key string, limit int64) ([]byte, error) { + obj, size, err := s.Get(ctx, key) + if err != nil { + return nil, err + } + defer obj.Close() + if size > limit { + return nil, fmt.Errorf("object larger than %d bytes", limit) + } + buf := bytes.NewBuffer(make([]byte, 0, size)) + if _, err := io.Copy(buf, io.LimitReader(obj, limit+1)); err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/src/internal/relay/relay.go b/src/internal/relay/relay.go new file mode 100644 index 00000000..722e3654 --- /dev/null +++ b/src/internal/relay/relay.go @@ -0,0 +1,73 @@ +package relay + +import ( + "context" + "log/slog" + "time" + + "mtusi-messenger/internal/broker" + "mtusi-messenger/internal/store" +) + +type Relay struct { + st *store.Store + pub *broker.Publisher + log *slog.Logger + kick chan struct{} + every time.Duration + failed bool +} + +func New(st *store.Store, pub *broker.Publisher, log *slog.Logger, every time.Duration) *Relay { + return &Relay{st: st, pub: pub, log: log, kick: make(chan struct{}, 1), every: every} +} + +func (r *Relay) Kick() { + select { + case r.kick <- struct{}{}: + default: + } +} + +func (r *Relay) Run(ctx context.Context) { + t := time.NewTicker(r.every) + defer t.Stop() + for { + r.drain(ctx) + select { + case <-ctx.Done(): + return + case <-t.C: + case <-r.kick: + } + } +} + +func (r *Relay) drain(ctx context.Context) { + for ctx.Err() == nil { + pctx, cancel := context.WithTimeout(ctx, 15*time.Second) + n, err := r.st.RelayOutbox(pctx, 50, func(row store.OutboxRow) error { + c, cc := context.WithTimeout(pctx, 5*time.Second) + defer cc() + return r.pub.Publish(c, row.JobID, row.Payload) + }) + cancel() + if err != nil { + if !r.failed { + r.log.Warn("outbox relay pass failed", "error", err.Error()) + } + r.failed = true + return + } + if r.failed { + r.log.Info("outbox relay recovered") + r.failed = false + } + if n > 0 { + r.log.Debug("outbox published", "count", n) + } + if n < 50 { + return + } + } +} diff --git a/src/internal/store/attachments.go b/src/internal/store/attachments.go new file mode 100644 index 00000000..f08a3501 --- /dev/null +++ b/src/internal/store/attachments.go @@ -0,0 +1,88 @@ +package store + +import ( + "context" + "errors" + "time" + + "github.com/jackc/pgx/v5" + + "mtusi-messenger/internal/model" +) + +const attachmentColumns = `a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, + a.object_key, a.status, a.error_code, a.pipeline_version, a.created_at, a.updated_at` + +func scanAttachment(row pgx.Row) (*model.Attachment, error) { + a := &model.Attachment{Variants: []model.Variant{}} + var created, updated time.Time + err := row.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256, + &a.ObjectKey, &a.Status, &a.ErrorCode, &a.PipelineVersion, &created, &updated) + if err != nil { + return nil, err + } + a.CreatedAt, a.UpdatedAt = utc(created), utc(updated) + return a, nil +} + +func (s *Store) InsertPendingAttachment(ctx context.Context, a *model.Attachment, width, height int, stagingKey string) error { + var created time.Time + err := s.pool.QueryRow(ctx, ` + INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + width, height, object_key, staging_key, status) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, 'pending') + RETURNING created_at`, + a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256, + width, height, a.ObjectKey, stagingKey).Scan(&created) + if pgCode(err) == "23503" { + return ErrNotFound + } + if err != nil { + return err + } + a.CreatedAt = utc(created) + return nil +} + +func (s *Store) DeletePendingAttachment(ctx context.Context, id string) (bool, error) { + tag, err := s.pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1 AND status = 'pending'`, id) + if err != nil { + return false, err + } + return tag.RowsAffected() == 1, nil +} + +func (s *Store) AttachmentForUser(ctx context.Context, id, userID string) (*model.Attachment, error) { + a, err := scanAttachment(s.pool.QueryRow(ctx, ` + SELECT `+attachmentColumns+` + FROM attachments a + WHERE a.id = $1 AND a.status <> 'pending' + AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = a.chat_id AND m.user_id = $2)`, + id, userID)) + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, err + } + if a.Status != "ready" { + return a, nil + } + rows, err := s.pool.Query(ctx, ` + SELECT name, content_type, size_bytes, width, height, sha256, object_key + FROM attachment_variants + WHERE attachment_id = $1 AND pipeline_version = $2 + ORDER BY CASE name WHEN 'thumbnail' THEN 1 WHEN 'crop' THEN 2 ELSE 3 END`, a.ID, a.PipelineVersion) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var v model.Variant + if err := rows.Scan(&v.Name, &v.ContentType, &v.SizeBytes, &v.Width, &v.Height, &v.SHA256, &v.ObjectKey); err != nil { + return nil, err + } + a.Variants = append(a.Variants, v) + } + return a, rows.Err() +} diff --git a/src/internal/store/cursor.go b/src/internal/store/cursor.go new file mode 100644 index 00000000..209405b2 --- /dev/null +++ b/src/internal/store/cursor.go @@ -0,0 +1,33 @@ +package store + +import ( + "encoding/base64" + "encoding/json" + "time" +) + +type Cursor struct { + Scope string `json:"s"` + TS time.Time `json:"t"` + ID string `json:"i"` +} + +func EncodeCursor(c Cursor) string { + b, _ := json.Marshal(c) + return base64.RawURLEncoding.EncodeToString(b) +} + +func DecodeCursor(s string) (Cursor, error) { + var c Cursor + b, err := base64.RawURLEncoding.DecodeString(s) + if err != nil { + return c, ErrInvalidCursor + } + if err := json.Unmarshal(b, &c); err != nil { + return c, ErrInvalidCursor + } + if c.Scope == "" || c.ID == "" || c.TS.IsZero() { + return c, ErrInvalidCursor + } + return c, nil +} diff --git a/src/internal/store/jobs.go b/src/internal/store/jobs.go new file mode 100644 index 00000000..d09f780e --- /dev/null +++ b/src/internal/store/jobs.go @@ -0,0 +1,493 @@ +package store + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "mtusi-messenger/internal/model" +) + +var ( + ErrJobNotFailed = errors.New("job is not in failed state") + ErrJobAlreadyOK = errors.New("job already completed") + ErrStagingGone = errors.New("staging original already removed by retention") + ErrQueueFull = errors.New("too much unfinished work") +) + +func (s *Store) inTx(ctx context.Context, fn func(pgx.Tx) error) error { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + if err := fn(tx); err != nil { + return err + } + return tx.Commit(ctx) +} + +func jobPayload(jobID, attID string, pv, attempt int, created time.Time) []byte { + b, _ := json.Marshal(model.JobMessage{ + SchemaVersion: 1, JobID: jobID, Type: model.JobType, AttachmentID: attID, + PipelineVersion: pv, Attempt: attempt, CreatedAt: created.UTC().Format(time.RFC3339Nano), + }) + return b +} + +func addOutbox(ctx context.Context, tx pgx.Tx, jobID, attID string, pv, attempt int, availableAt *time.Time) error { + _, err := tx.Exec(ctx, ` + INSERT INTO job_outbox (job_id, attempt, payload, available_at) + VALUES ($1, $2, $3, COALESCE($4, clock_timestamp()))`, + jobID, attempt, jobPayload(jobID, attID, pv, attempt, time.Now()), availableAt) + return err +} + +func addEvent(ctx context.Context, tx pgx.Tx, jobID string, attempt int, event string, code, detail *string, actor string) error { + _, err := tx.Exec(ctx, ` + INSERT INTO job_events (job_id, attempt, event, error_code, detail, actor) + VALUES ($1, $2, $3, $4, $5, NULLIF($6, ''))`, jobID, attempt, event, code, detail, actor) + return err +} + +func (s *Store) QueueAttachment(ctx context.Context, attID string, pipelineVersion, maxAttempts, maxActive int) (string, time.Time, error) { + jobID := uuid.NewString() + var updated time.Time + err := s.inTx(ctx, func(tx pgx.Tx) error { + if maxActive > 0 { + var active int + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(20260505)`); err != nil { + return err + } + if err := tx.QueryRow(ctx, `SELECT count(*) FROM attachment_jobs WHERE status IN ('queued', 'processing')`).Scan(&active); err != nil { + return err + } + if active >= maxActive { + return ErrQueueFull + } + } + err := tx.QueryRow(ctx, ` + UPDATE attachments SET status = 'queued', pipeline_version = $2, + updated_at = date_trunc('milliseconds', now()) + WHERE id = $1 AND status = 'pending' + RETURNING updated_at`, attID, pipelineVersion).Scan(&updated) + if errors.Is(err, pgx.ErrNoRows) { + return ErrNotFound + } + if err != nil { + return err + } + if _, err := tx.Exec(ctx, ` + INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts) + VALUES ($1, $2, $3, 'queued', 1, $4)`, jobID, attID, pipelineVersion, maxAttempts); err != nil { + return err + } + if err := addEvent(ctx, tx, jobID, 1, "queued", nil, nil, "api"); err != nil { + return err + } + return addOutbox(ctx, tx, jobID, attID, pipelineVersion, 1, nil) + }) + return jobID, updated.UTC(), err +} + +func (s *Store) ActiveJobs(ctx context.Context) (int, error) { + var n int + err := s.pool.QueryRow(ctx, `SELECT count(*) FROM attachment_jobs WHERE status IN ('queued', 'processing')`).Scan(&n) + return n, err +} + +type OutboxRow struct { + ID int64 + JobID string + Attempt int + Payload []byte +} + +func (s *Store) RelayOutbox(ctx context.Context, limit int, publish func(OutboxRow) error) (int, error) { + published := 0 + err := s.inTx(ctx, func(tx pgx.Tx) error { + rows, err := tx.Query(ctx, ` + SELECT id, job_id, attempt, payload FROM job_outbox + WHERE published_at IS NULL AND available_at <= clock_timestamp() + ORDER BY available_at, id + LIMIT $1 + FOR UPDATE SKIP LOCKED`, limit) + if err != nil { + return err + } + var batch []OutboxRow + for rows.Next() { + var r OutboxRow + if err := rows.Scan(&r.ID, &r.JobID, &r.Attempt, &r.Payload); err != nil { + rows.Close() + return err + } + batch = append(batch, r) + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + for _, r := range batch { + if perr := publish(r); perr != nil { + msg := perr.Error() + _, _ = tx.Exec(ctx, `UPDATE job_outbox SET publish_attempts = publish_attempts + 1, last_error = $2 WHERE id = $1`, r.ID, msg) + return nil + } + if _, err := tx.Exec(ctx, ` + UPDATE job_outbox SET published_at = clock_timestamp(), publish_attempts = publish_attempts + 1, last_error = NULL + WHERE id = $1`, r.ID); err != nil { + return err + } + published++ + } + return nil + }) + if err != nil { + return 0, err + } + return published, nil +} + +func (s *Store) PurgeOutbox(ctx context.Context, olderThan time.Duration) (int64, error) { + tag, err := s.pool.Exec(ctx, `DELETE FROM job_outbox WHERE published_at < now() - make_interval(secs => $1)`, olderThan.Seconds()) + return tag.RowsAffected(), err +} + +type JobWork struct { + JobID string + AttachmentID string + ChatID string + Filename string + ContentType string + SizeBytes int64 + SHA256 string + ObjectKey string + StagingKey string + PipelineVersion int + Attempt int + MaxAttempts int +} + +type ClaimResult string + +const ( + Claimed ClaimResult = "claimed" + Done ClaimResult = "done" + Stale ClaimResult = "stale" + Busy ClaimResult = "busy" + NotFound ClaimResult = "notfound" + NotYet ClaimResult = "notyet" +) + +func (s *Store) ClaimJob(ctx context.Context, jobID string, attempt int, owner string, lease time.Duration, takeover bool) (*JobWork, ClaimResult, error) { + var w JobWork + result := Claimed + err := s.inTx(ctx, func(tx pgx.Tx) error { + var status string + var curAttempt int + var next *time.Time + err := tx.QueryRow(ctx, `SELECT status, attempt, next_attempt_at FROM attachment_jobs WHERE job_id = $1 FOR UPDATE`, jobID). + Scan(&status, &curAttempt, &next) + if errors.Is(err, pgx.ErrNoRows) { + result = NotFound + return nil + } + if err != nil { + return err + } + switch { + case status == "ready" || status == "failed": + result = Done + return nil + case curAttempt != attempt: + result = Stale + return nil + case status == "processing" && !takeover: + var expired bool + if err := tx.QueryRow(ctx, `SELECT lease_until < clock_timestamp() FROM attachment_jobs WHERE job_id = $1`, jobID).Scan(&expired); err != nil { + return err + } + if !expired { + result = Busy + return nil + } + case status == "queued" && next != nil && next.After(time.Now().Add(2*time.Second)): + result = NotYet + return nil + } + err = tx.QueryRow(ctx, ` + UPDATE attachment_jobs + SET status = 'processing', lease_owner = $2, + lease_until = clock_timestamp() + make_interval(secs => $3), + started_at = clock_timestamp(), updated_at = clock_timestamp() + WHERE job_id = $1 + RETURNING attachment_id, pipeline_version, attempt, max_attempts`, jobID, owner, lease.Seconds()). + Scan(&w.AttachmentID, &w.PipelineVersion, &w.Attempt, &w.MaxAttempts) + if err != nil { + return err + } + err = tx.QueryRow(ctx, ` + UPDATE attachments SET status = 'processing', updated_at = date_trunc('milliseconds', now()) + WHERE id = $1 AND status IN ('queued', 'processing') + RETURNING chat_id, filename, content_type, size_bytes, sha256, object_key, COALESCE(staging_key, '')`, w.AttachmentID). + Scan(&w.ChatID, &w.Filename, &w.ContentType, &w.SizeBytes, &w.SHA256, &w.ObjectKey, &w.StagingKey) + if errors.Is(err, pgx.ErrNoRows) { + result = NotFound + return errors.New("rollback") + } + if err != nil { + return err + } + ev := "started" + if status == "processing" { + ev = "taken_over" + } + return addEvent(ctx, tx, jobID, attempt, ev, nil, nil, owner) + }) + if result == NotFound { + return nil, NotFound, nil + } + if err != nil { + return nil, "", err + } + if result != Claimed { + return nil, result, nil + } + w.JobID = jobID + return &w, Claimed, nil +} + +func (s *Store) ExtendLease(ctx context.Context, jobID string, attempt int, owner string, lease time.Duration) (bool, error) { + tag, err := s.pool.Exec(ctx, ` + UPDATE attachment_jobs SET lease_until = clock_timestamp() + make_interval(secs => $4) + WHERE job_id = $1 AND attempt = $2 AND lease_owner = $3 AND status = 'processing'`, + jobID, attempt, owner, lease.Seconds()) + return tag.RowsAffected() == 1, err +} + +func (s *Store) CompleteJob(ctx context.Context, w *JobWork, variants []model.Variant) (bool, error) { + ok := false + err := s.inTx(ctx, func(tx pgx.Tx) error { + tag, err := tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status = 'ready', lease_owner = NULL, lease_until = NULL, error_code = NULL, + finished_at = clock_timestamp(), updated_at = clock_timestamp() + WHERE job_id = $1 AND attempt = $2 AND status = 'processing'`, w.JobID, w.Attempt) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return nil + } + for _, v := range variants { + if _, err := tx.Exec(ctx, ` + INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, + size_bytes, width, height, sha256, object_key) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING`, + w.AttachmentID, w.PipelineVersion, v.Name, v.ContentType, v.SizeBytes, v.Width, v.Height, v.SHA256, v.ObjectKey); err != nil { + return err + } + } + if _, err := tx.Exec(ctx, ` + UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = date_trunc('milliseconds', now()) + WHERE id = $1 AND status = 'processing'`, w.AttachmentID); err != nil { + return err + } + ok = true + return addEvent(ctx, tx, w.JobID, w.Attempt, "ready", nil, nil, "") + }) + return ok, err +} + +func (s *Store) ClearStaging(ctx context.Context, attID string) error { + _, err := s.pool.Exec(ctx, `UPDATE attachments SET staging_key = NULL WHERE id = $1 AND status IN ('ready', 'failed')`, attID) + return err +} + +func (s *Store) FailOrRetry(ctx context.Context, jobID string, attempt int, code, detail string, permanent bool, backoff time.Duration, actor string) (string, error) { + out := "" + err := s.inTx(ctx, func(tx pgx.Tx) error { + var attID string + var pv, maxAttempts int + err := tx.QueryRow(ctx, ` + SELECT attachment_id, pipeline_version, max_attempts FROM attachment_jobs + WHERE job_id = $1 AND attempt = $2 AND status = 'processing' FOR UPDATE`, jobID, attempt). + Scan(&attID, &pv, &maxAttempts) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return err + } + return failOrRetryLocked(ctx, tx, jobID, attID, pv, attempt, maxAttempts, code, detail, permanent, backoff, actor, &out) + }) + return out, err +} + +func failOrRetryLocked(ctx context.Context, tx pgx.Tx, jobID, attID string, pv, attempt, maxAttempts int, code, detail string, permanent bool, backoff time.Duration, actor string, out *string) error { + if !permanent && attempt >= maxAttempts { + permanent = true + detail = code + ": " + detail + code = "retry_exhausted" + } + if permanent { + if _, err := tx.Exec(ctx, ` + UPDATE attachment_jobs SET status = 'failed', error_code = $2, last_error = $3, + lease_owner = NULL, lease_until = NULL, finished_at = clock_timestamp(), updated_at = clock_timestamp() + WHERE job_id = $1`, jobID, code, detail); err != nil { + return err + } + if _, err := tx.Exec(ctx, ` + UPDATE attachments SET status = 'failed', error_code = $2, updated_at = date_trunc('milliseconds', now()) + WHERE id = $1 AND status IN ('queued', 'processing')`, attID, code); err != nil { + return err + } + *out = "failed" + return addEvent(ctx, tx, jobID, attempt, "failed", &code, &detail, actor) + } + next := time.Now().Add(backoff) + if _, err := tx.Exec(ctx, ` + UPDATE attachment_jobs SET status = 'queued', attempt = attempt + 1, next_attempt_at = $2, + error_code = $3, last_error = $4, lease_owner = NULL, lease_until = NULL, updated_at = clock_timestamp() + WHERE job_id = $1`, jobID, next, code, detail); err != nil { + return err + } + if _, err := tx.Exec(ctx, ` + UPDATE attachments SET status = 'queued', updated_at = date_trunc('milliseconds', now()) + WHERE id = $1 AND status IN ('queued', 'processing')`, attID); err != nil { + return err + } + if err := addEvent(ctx, tx, jobID, attempt, "retry_scheduled", &code, &detail, actor); err != nil { + return err + } + *out = "queued" + return addOutbox(ctx, tx, jobID, attID, pv, attempt+1, &next) +} + +func (s *Store) RecoverExpiredLeases(ctx context.Context, limit int) (int, error) { + n := 0 + err := s.inTx(ctx, func(tx pgx.Tx) error { + rows, err := tx.Query(ctx, ` + SELECT job_id, attachment_id, pipeline_version, attempt, max_attempts FROM attachment_jobs + WHERE status = 'processing' AND lease_until < clock_timestamp() + ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED`, limit) + if err != nil { + return err + } + type rec struct { + job, att string + pv, attempt, maxAt int + } + var list []rec + for rows.Next() { + var r rec + if err := rows.Scan(&r.job, &r.att, &r.pv, &r.attempt, &r.maxAt); err != nil { + rows.Close() + return err + } + list = append(list, r) + } + rows.Close() + for _, r := range list { + var out string + if err := failOrRetryLocked(ctx, tx, r.job, r.att, r.pv, r.attempt, r.maxAt, "lease_expired", + "worker lease expired", false, 0, "sweeper", &out); err != nil { + return err + } + n++ + } + return nil + }) + return n, err +} + +func (s *Store) ReplayFailed(ctx context.Context, attID, actor string, extraAttempts int) (string, int, error) { + var jobID string + var attempt int + err := s.inTx(ctx, func(tx pgx.Tx) error { + var status string + var pv int + var staging *string + err := tx.QueryRow(ctx, ` + SELECT j.job_id, j.status, j.attempt, j.pipeline_version, a.staging_key + FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id + WHERE j.attachment_id = $1 ORDER BY j.pipeline_version DESC LIMIT 1 + FOR UPDATE`, attID).Scan(&jobID, &status, &attempt, &pv, &staging) + if errors.Is(err, pgx.ErrNoRows) { + return ErrNotFound + } + if err != nil { + return err + } + switch status { + case "ready": + return ErrJobAlreadyOK + case "failed": + default: + return ErrJobNotFailed + } + if staging == nil { + return ErrStagingGone + } + attempt++ + if _, err := tx.Exec(ctx, ` + UPDATE attachment_jobs SET status = 'queued', attempt = $2, max_attempts = $3, next_attempt_at = NULL, + error_code = NULL, finished_at = NULL, updated_at = clock_timestamp() + WHERE job_id = $1`, jobID, attempt, attempt+extraAttempts-1); err != nil { + return err + } + if _, err := tx.Exec(ctx, ` + UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = date_trunc('milliseconds', now()) + WHERE id = $1 AND status = 'failed'`, attID); err != nil { + return err + } + if err := addEvent(ctx, tx, jobID, attempt, "replay", nil, nil, actor); err != nil { + return err + } + return addOutbox(ctx, tx, jobID, attID, pv, attempt, nil) + }) + return jobID, attempt, err +} + +type StagingRef struct { + AttachmentID string + StagingKey string + Status string +} + +func (s *Store) StaleStaging(ctx context.Context, pendingGrace, failedRetention time.Duration, limit int) ([]StagingRef, error) { + rows, err := s.pool.Query(ctx, ` + SELECT id, staging_key, status FROM attachments + WHERE staging_key IS NOT NULL + AND ((status = 'pending' AND created_at < now() - make_interval(secs => $1)) + OR (status = 'failed' AND updated_at < now() - make_interval(secs => $2))) + LIMIT $3`, pendingGrace.Seconds(), failedRetention.Seconds(), limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []StagingRef + for rows.Next() { + var r StagingRef + if err := rows.Scan(&r.AttachmentID, &r.StagingKey, &r.Status); err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +func (s *Store) ForgetStaging(ctx context.Context, r StagingRef) error { + if r.Status == "pending" { + _, err := s.pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1 AND status = 'pending'`, r.AttachmentID) + return err + } + _, err := s.pool.Exec(ctx, `UPDATE attachments SET staging_key = NULL WHERE id = $1 AND status = 'failed'`, r.AttachmentID) + return err +} diff --git a/src/internal/store/postgres.go b/src/internal/store/postgres.go new file mode 100644 index 00000000..61be606d --- /dev/null +++ b/src/internal/store/postgres.go @@ -0,0 +1,388 @@ +package store + +import ( + "bytes" + "context" + "errors" + "net" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "mtusi-messenger/internal/model" +) + +var ( + ErrNotFound = errors.New("not found") + ErrUsernameTaken = errors.New("username taken") + ErrUnknownMember = errors.New("unknown member") + ErrInvalidCursor = errors.New("invalid cursor") +) + +type Store struct { + pool *pgxpool.Pool +} + +func New(pool *pgxpool.Pool) *Store { return &Store{pool: pool} } + +func (s *Store) Ping(ctx context.Context) error { return s.pool.Ping(ctx) } + +func IsUnavailable(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) || pgconn.Timeout(err) { + return true + } + var connErr *pgconn.ConnectError + if errors.As(err, &connErr) { + return true + } + var netErr net.Error + if errors.As(err, &netErr) { + return true + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + + switch { + case pgErr.Code == "57P01", pgErr.Code == "57P03", pgErr.Code == "57014", pgErr.Code == "55P03", + len(pgErr.Code) == 5 && pgErr.Code[:2] == "08": + return true + } + } + return errors.Is(err, pgx.ErrTxClosed) || errors.Is(err, net.ErrClosed) +} + +func pgCode(err error) string { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + return pgErr.Code + } + return "" +} + +func utc(t time.Time) model.Time { return model.NewTime(t.UTC()) } + +func (s *Store) CreateUser(ctx context.Context, username, displayName, passwordHash string) (*model.User, error) { + u := &model.User{ID: uuid.NewString(), Username: username, DisplayName: displayName} + var created time.Time + err := s.pool.QueryRow(ctx, + `INSERT INTO users (id, username, display_name, password_hash) + VALUES ($1, $2, $3, $4) RETURNING created_at`, + u.ID, username, displayName, passwordHash).Scan(&created) + if pgCode(err) == "23505" { + return nil, ErrUsernameTaken + } + if err != nil { + return nil, err + } + u.CreatedAt = utc(created) + return u, nil +} + +func (s *Store) Credentials(ctx context.Context, username string) (*model.User, string, error) { + u := &model.User{} + var created time.Time + var hash string + err := s.pool.QueryRow(ctx, + `SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = $1`, + username).Scan(&u.ID, &u.Username, &u.DisplayName, &created, &hash) + if errors.Is(err, pgx.ErrNoRows) { + return nil, "", ErrNotFound + } + if err != nil { + return nil, "", err + } + u.CreatedAt = utc(created) + return u, hash, nil +} + +func (s *Store) CreateSession(ctx context.Context, tokenHash []byte, userID string, ttl time.Duration) (time.Time, error) { + var expires time.Time + err := s.pool.QueryRow(ctx, + `INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES ($1, $2, date_trunc('milliseconds', now()) + make_interval(secs => $3)) + RETURNING expires_at`, + tokenHash, userID, ttl.Seconds()).Scan(&expires) + if err != nil { + return time.Time{}, err + } + return expires.UTC(), nil +} + +func (s *Store) SessionUser(ctx context.Context, tokenHash []byte) (*model.User, time.Time, error) { + u := &model.User{} + var created, expires time.Time + err := s.pool.QueryRow(ctx, + `SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at + FROM sessions s JOIN users u ON u.id = s.user_id + WHERE s.token_hash = $1 AND s.revoked_at IS NULL AND s.expires_at > now()`, + tokenHash).Scan(&u.ID, &u.Username, &u.DisplayName, &created, &expires) + if errors.Is(err, pgx.ErrNoRows) { + return nil, time.Time{}, ErrNotFound + } + if err != nil { + return nil, time.Time{}, err + } + u.CreatedAt = utc(created) + return u, expires.UTC(), nil +} + +func (s *Store) RevokeSession(ctx context.Context, tokenHash []byte) error { + tag, err := s.pool.Exec(ctx, + `UPDATE sessions SET revoked_at = now() + WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > now()`, tokenHash) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + return ErrNotFound + } + return nil +} + +func (s *Store) PurgeSessions(ctx context.Context, olderThan time.Duration) (int64, error) { + tag, err := s.pool.Exec(ctx, + `DELETE FROM sessions + WHERE expires_at < now() - make_interval(secs => $1) + OR revoked_at < now() - make_interval(secs => $1)`, olderThan.Seconds()) + if err != nil { + return 0, err + } + return tag.RowsAffected(), nil +} + +func (s *Store) CreateChat(ctx context.Context, creatorID, title string, memberIDs []string) (*model.Chat, error) { + c := &model.Chat{ID: uuid.NewString(), Title: title, MemberIDs: append([]string(nil), memberIDs...)} + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + var created time.Time + if err := tx.QueryRow(ctx, + `INSERT INTO chats (id, title, created_by) VALUES ($1, $2, $3) RETURNING created_at`, + c.ID, title, creatorID).Scan(&created); err != nil { + return err + } + c.CreatedAt = utc(created) + _, err := tx.Exec(ctx, + `INSERT INTO chat_members (chat_id, user_id, position) + SELECT $1, m.user_id, m.ord - 1 + FROM unnest($2::uuid[]) WITH ORDINALITY AS m(user_id, ord)`, + c.ID, memberIDs) + return err + }) + if pgCode(err) == "23503" { + return nil, ErrUnknownMember + } + if err != nil { + return nil, err + } + return c, nil +} + +func (s *Store) IsMember(ctx context.Context, chatID, userID string) (bool, error) { + var ok bool + err := s.pool.QueryRow(ctx, + `SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, + chatID, userID).Scan(&ok) + return ok, err +} + +func (s *Store) ChatForUser(ctx context.Context, chatID, userID string) (*model.Chat, error) { + c := &model.Chat{} + var created time.Time + err := s.pool.QueryRow(ctx, + `SELECT c.id, c.title, c.created_at, + array_agg(m.user_id::text ORDER BY m.position) + FROM chats c JOIN chat_members m ON m.chat_id = c.id + WHERE c.id = $1 + AND EXISTS (SELECT 1 FROM chat_members x WHERE x.chat_id = c.id AND x.user_id = $2) + GROUP BY c.id`, + chatID, userID).Scan(&c.ID, &c.Title, &created, &c.MemberIDs) + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, err + } + c.CreatedAt = utc(created) + return c, nil +} + +func (s *Store) ListChats(ctx context.Context, userID string, limit int, cur *Cursor) ([]*model.Chat, bool, error) { + var ts *time.Time + var cid *string + if cur != nil { + ts, cid = &cur.TS, &cur.ID + } + rows, err := s.pool.Query(ctx, + `SELECT c.id, c.title, c.created_at, + (SELECT array_agg(m.user_id::text ORDER BY m.position) FROM chat_members m WHERE m.chat_id = c.id) + FROM chat_members me JOIN chats c ON c.id = me.chat_id + WHERE me.user_id = $1 + AND ($2::timestamptz IS NULL OR (c.created_at, c.id) > ($2, $3::uuid)) + ORDER BY c.created_at, c.id + LIMIT $4`, + userID, ts, cid, limit+1) + if err != nil { + return nil, false, err + } + defer rows.Close() + out := make([]*model.Chat, 0, limit+1) + for rows.Next() { + c := &model.Chat{} + var created time.Time + if err := rows.Scan(&c.ID, &c.Title, &created, &c.MemberIDs); err != nil { + return nil, false, err + } + c.CreatedAt = utc(created) + out = append(out, c) + } + if err := rows.Err(); err != nil { + return nil, false, err + } + more := len(out) > limit + if more { + out = out[:limit] + } + return out, more, nil +} + +var ( + ErrIdempotencyConflict = errors.New("idempotency key reused with different body") + ErrAttachmentNotFound = errors.New("attachment not found in chat") + ErrAttachmentFailed = errors.New("attachment processing failed") +) + +func (s *Store) AddMessage(ctx context.Context, chatID, senderID, text string, attachmentIDs []string, key string, reqHash []byte, ttl time.Duration) (*model.Message, bool, error) { + for attempt := 0; attempt < 2; attempt++ { + newID := uuid.NewString() + var member, attachmentsOK bool + var failedCount int64 + var created *time.Time + err := s.pool.QueryRow(ctx, ` + WITH member AS ( + SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $1 + ), att AS ( + SELECT count(*) FILTER (WHERE status IN ('queued', 'processing', 'ready')) = cardinality($8::uuid[]) AS ok, + count(*) FILTER (WHERE status = 'failed') AS failed + FROM attachments + WHERE chat_id = $2 AND id = ANY($8::uuid[]) + ), k AS ( + INSERT INTO message_idempotency (user_id, chat_id, idem_key, request_hash, message_id) + SELECT $1, $2, $3, $4, $5 FROM member, att WHERE att.ok + ON CONFLICT (user_id, chat_id, idem_key) DO UPDATE + SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = now() + WHERE message_idempotency.created_at < now() - make_interval(secs => $7) + RETURNING message_id + ), m AS ( + INSERT INTO messages (id, chat_id, sender_id, text) + SELECT $5, $2, $1, $6 FROM k + RETURNING created_at + ), ma AS ( + INSERT INTO message_attachments (message_id, chat_id, attachment_id, position) + SELECT $5, $2, a.id, a.ord - 1 + FROM unnest($8::uuid[]) WITH ORDINALITY AS a(id, ord) + WHERE EXISTS (SELECT 1 FROM m) + ) + SELECT EXISTS (SELECT 1 FROM member), (SELECT ok FROM att), (SELECT failed FROM att), (SELECT created_at FROM m)`, + senderID, chatID, key, reqHash, newID, text, ttl.Seconds(), attachmentIDs).Scan(&member, &attachmentsOK, &failedCount, &created) + if pgCode(err) == "23503" { + return nil, false, ErrAttachmentNotFound + } + if err != nil { + return nil, false, err + } + if !member { + return nil, false, ErrNotFound + } + if !attachmentsOK { + if failedCount > 0 { + return nil, false, ErrAttachmentFailed + } + return nil, false, ErrAttachmentNotFound + } + if created != nil { + return &model.Message{ID: newID, ChatID: chatID, SenderID: senderID, Text: text, + CreatedAt: utc(*created), AttachmentIDs: attachmentIDs}, false, nil + } + var stored []byte + var at time.Time + m := &model.Message{ChatID: chatID} + err = s.pool.QueryRow(ctx, ` + SELECT i.request_hash, m.id, m.sender_id, m.text, m.created_at, + `+attachmentIDsSQL+` + FROM message_idempotency i JOIN messages m ON m.id = i.message_id + WHERE i.user_id = $1 AND i.chat_id = $2 AND i.idem_key = $3 + AND i.created_at >= now() - make_interval(secs => $4)`, + senderID, chatID, key, ttl.Seconds()).Scan(&stored, &m.ID, &m.SenderID, &m.Text, &at, &m.AttachmentIDs) + if errors.Is(err, pgx.ErrNoRows) { + continue + } + if err != nil { + return nil, false, err + } + if !bytes.Equal(stored, reqHash) { + return nil, false, ErrIdempotencyConflict + } + m.CreatedAt = utc(at) + return m, true, nil + } + return nil, false, errors.New("idempotency record kept changing") +} + +const attachmentIDsSQL = `COALESCE((SELECT array_agg(ma.attachment_id::text ORDER BY ma.position) + FROM message_attachments ma WHERE ma.message_id = m.id), '{}')` + +func (s *Store) PurgeIdempotency(ctx context.Context, ttl time.Duration) (int64, error) { + tag, err := s.pool.Exec(ctx, + `DELETE FROM message_idempotency WHERE created_at < now() - make_interval(secs => $1)`, ttl.Seconds()) + if err != nil { + return 0, err + } + return tag.RowsAffected(), nil +} + +func (s *Store) ListMessages(ctx context.Context, chatID string, limit int, cur *Cursor) ([]*model.Message, bool, error) { + var ( + rows pgx.Rows + err error + ) + if cur == nil { + rows, err = s.pool.Query(ctx, + `SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, `+attachmentIDsSQL+` + FROM messages m + WHERE m.chat_id = $1 + ORDER BY m.created_at, m.id LIMIT $2`, chatID, limit+1) + } else { + rows, err = s.pool.Query(ctx, + `SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, `+attachmentIDsSQL+` + FROM messages m + WHERE m.chat_id = $1 AND (m.created_at, m.id) > ($2, $3::uuid) + ORDER BY m.created_at, m.id LIMIT $4`, chatID, cur.TS, cur.ID, limit+1) + } + if err != nil { + return nil, false, err + } + defer rows.Close() + out := make([]*model.Message, 0, limit+1) + for rows.Next() { + m := &model.Message{} + var created time.Time + if err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &created, &m.AttachmentIDs); err != nil { + return nil, false, err + } + m.CreatedAt = utc(created) + out = append(out, m) + } + if err := rows.Err(); err != nil { + return nil, false, err + } + more := len(out) > limit + if more { + out = out[:limit] + } + return out, more, nil +} diff --git a/src/internal/worker/worker.go b/src/internal/worker/worker.go new file mode 100644 index 00000000..0838a3d4 --- /dev/null +++ b/src/internal/worker/worker.go @@ -0,0 +1,365 @@ +package worker + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log/slog" + "math/rand/v2" + "os" + "strings" + "sync" + "time" + + "github.com/google/uuid" + amqp "github.com/rabbitmq/amqp091-go" + + "mtusi-messenger/internal/broker" + "mtusi-messenger/internal/imaging" + "mtusi-messenger/internal/model" + "mtusi-messenger/internal/objstore" + "mtusi-messenger/internal/store" +) + +type Config struct { + BrokerURL string + Concurrency int + MaxAttempts int + JobTimeout time.Duration + Lease time.Duration + BackoffBase time.Duration + BackoffMax time.Duration + StagingRetain time.Duration + PendingGrace time.Duration + FaultInjection bool + Limits imaging.Limits + SweepEvery time.Duration +} + +type Worker struct { + cfg Config + st *store.Store + s3 *objstore.Store + log *slog.Logger + owner string + inFly sync.WaitGroup +} + +func New(cfg Config, st *store.Store, s3 *objstore.Store, log *slog.Logger) *Worker { + host, _ := os.Hostname() + return &Worker{cfg: cfg, st: st, s3: s3, log: log, owner: host + "/" + uuid.NewString()[:8]} +} + +func (w *Worker) Run(ctx context.Context) error { + go w.sweeper(ctx) + for ctx.Err() == nil { + if err := w.consume(ctx); err != nil && ctx.Err() == nil { + w.log.Warn("consumer stopped, reconnecting", "error", err.Error()) + select { + case <-ctx.Done(): + case <-time.After(2 * time.Second): + } + } + } + w.inFly.Wait() + return nil +} + +func (w *Worker) consume(ctx context.Context) error { + conn, err := broker.Dial(w.cfg.BrokerURL) + if err != nil { + return err + } + defer conn.Close() + ch, err := conn.Channel() + if err != nil { + return err + } + if err := broker.Declare(ch); err != nil { + return err + } + + if err := ch.Qos(w.cfg.Concurrency, 0, false); err != nil { + return err + } + deliveries, err := ch.ConsumeWithContext(ctx, broker.Queue, w.owner, false, false, false, false, nil) + if err != nil { + return err + } + w.log.Info("worker consuming", "queue", broker.Queue, "concurrency", w.cfg.Concurrency, "owner", w.owner) + closed := conn.NotifyClose(make(chan *amqp.Error, 1)) + var wg sync.WaitGroup + for i := 0; i < w.cfg.Concurrency; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for d := range deliveries { + w.inFly.Add(1) + w.handle(ctx, d) + w.inFly.Done() + } + }() + } + select { + case <-ctx.Done(): + _ = ch.Cancel(w.owner, false) + wg.Wait() + return nil + case e := <-closed: + wg.Wait() + if e == nil { + return errors.New("broker connection closed") + } + return e + } +} + +func (w *Worker) fault(job *store.JobWork, point string) bool { + return w.cfg.FaultInjection && strings.HasPrefix(job.Filename, "fault-"+point) +} + +func (w *Worker) handle(ctx context.Context, d amqp.Delivery) { + var msg model.JobMessage + if err := json.Unmarshal(d.Body, &msg); err != nil || msg.SchemaVersion != 1 || msg.Type != model.JobType || + uuid.Validate(msg.JobID) != nil || uuid.Validate(msg.AttachmentID) != nil || msg.Attempt < 1 || msg.PipelineVersion < 1 { + w.log.Error("malformed job message -> DLQ", "message_id", d.MessageId) + _ = d.Nack(false, false) + return + } + log := w.log.With("job_id", msg.JobID, "attachment_id", msg.AttachmentID, "attempt", msg.Attempt, "redelivered", d.Redelivered) + cctx, cancel := context.WithTimeout(ctx, 10*time.Second) + job, res, err := w.st.ClaimJob(cctx, msg.JobID, msg.Attempt, w.owner, w.cfg.Lease, d.Redelivered) + cancel() + if err != nil { + log.Warn("claim failed, requeue", "error", err.Error()) + time.Sleep(2 * time.Second) + _ = d.Nack(false, true) + return + } + if res != store.Claimed { + + log.Info("job delivery is a no-op", "result", string(res)) + _ = d.Ack(false) + return + } + if job.AttachmentID != msg.AttachmentID { + log.Error("message attachment_id does not match job") + _ = d.Ack(false) + return + } + + if w.fault(job, "hang") && job.Attempt == 1 { + + log.Warn("FAULT: hang without heartbeat", "for", (3 * w.cfg.Lease).String()) + time.Sleep(3 * w.cfg.Lease) + } + + start := time.Now() + variants, perr := w.process(ctx, job, d.Redelivered, log) + if perr == nil { + cctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ok, err := w.st.CompleteJob(cctx, job, variants) + cancel() + if err != nil { + log.Warn("result commit failed, requeue", "error", err.Error()) + time.Sleep(2 * time.Second) + _ = d.Nack(false, true) + return + } + if !ok { + log.Info("stale attempt finished after takeover; result already owned by newer attempt") + } else { + log.Info("job ready", "ms", time.Since(start).Milliseconds()) + w.dropStaging(job, log) + } + if w.fault(job, "worker-before-ack") && !d.Redelivered { + log.Warn("FAULT: exit after COMMIT before ack") + os.Exit(42) + } + _ = d.Ack(false) + return + } + + code, permanent := classify(perr) + backoff := w.backoff(job.Attempt) + cctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) + status, err := w.st.FailOrRetry(cctx, job.JobID, job.Attempt, code, safeDetail(perr), permanent, backoff, w.owner) + cancel() + if err != nil { + log.Warn("failure bookkeeping failed, requeue", "error", err.Error()) + time.Sleep(2 * time.Second) + _ = d.Nack(false, true) + return + } + log.Warn("job attempt failed", "error_code", code, "permanent", permanent, "next", status, "backoff_ms", backoff.Milliseconds()) + _ = d.Ack(false) +} + +func (w *Worker) process(ctx context.Context, job *store.JobWork, redelivered bool, log *slog.Logger) ([]model.Variant, error) { + pctx, cancel := context.WithTimeout(ctx, w.cfg.JobTimeout) + defer cancel() + + hbDone := make(chan struct{}) + defer close(hbDone) + go func() { + t := time.NewTicker(w.cfg.Lease / 3) + defer t.Stop() + for { + select { + case <-hbDone: + return + case <-pctx.Done(): + return + case <-t.C: + c, cc := context.WithTimeout(context.Background(), 3*time.Second) + _, _ = w.st.ExtendLease(c, job.JobID, job.Attempt, w.owner, w.cfg.Lease) + cc() + } + } + }() + + if w.fault(job, "flaky") && job.Attempt <= w.cfg.MaxAttempts { + return nil, transientErr("injected_transient", "flaky fault until replay") + } + if w.fault(job, "transient") && job.Attempt < 3 { + return nil, transientErr("injected_transient", "first two attempts fail") + } + + data, err := w.readOriginal(pctx, job) + if err != nil { + return nil, err + } + sum := sha256.Sum256(data) + if hex.EncodeToString(sum[:]) != job.SHA256 || int64(len(data)) != job.SizeBytes { + return nil, &imaging.PermanentError{Code: "processing_failed", Detail: "staged original checksum mismatch"} + } + outs, err := imaging.Process(pctx, data, job.ContentType, w.cfg.Limits) + if err != nil { + return nil, err + } + + if err := w.s3.Put(pctx, job.ObjectKey, data, job.ContentType); err != nil { + return nil, transientErr("storage_unavailable", err.Error()) + } + variants := make([]model.Variant, 0, len(outs)) + for _, o := range outs { + key := fmt.Sprintf("variants/%s/%s/v%d/%s.%s", job.ChatID, job.AttachmentID, job.PipelineVersion, o.Name, o.Ext) + if err := w.s3.Put(pctx, key, o.Data, o.ContentType); err != nil { + return nil, transientErr("storage_unavailable", err.Error()) + } + vs := sha256.Sum256(o.Data) + variants = append(variants, model.Variant{Name: o.Name, ContentType: o.ContentType, SizeBytes: int64(len(o.Data)), + Width: o.Width, Height: o.Height, SHA256: hex.EncodeToString(vs[:]), ObjectKey: key}) + } + if w.fault(job, "worker-after-effect") && !redelivered { + log.Warn("FAULT: exit after S3 writes before COMMIT") + os.Exit(42) + } + return variants, nil +} + +func (w *Worker) readOriginal(ctx context.Context, job *store.JobWork) ([]byte, error) { + if job.StagingKey != "" { + data, err := w.s3.ReadAll(ctx, job.StagingKey, w.cfg.Limits.MaxBytes) + if err == nil { + return data, nil + } + if !objstore.IsNotFound(err) { + return nil, transientErr("storage_unavailable", err.Error()) + } + } + data, err := w.s3.ReadAll(ctx, job.ObjectKey, w.cfg.Limits.MaxBytes) + if objstore.IsNotFound(err) { + return nil, &imaging.PermanentError{Code: "processing_failed", Detail: "original is missing"} + } + if err != nil { + return nil, transientErr("storage_unavailable", err.Error()) + } + return data, nil +} + +func (w *Worker) dropStaging(job *store.JobWork, log *slog.Logger) { + if job.StagingKey == "" { + return + } + c, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := w.s3.Remove(c, job.StagingKey); err != nil { + log.Warn("staging cleanup deferred to sweeper", "error", err.Error()) + return + } + _ = w.st.ClearStaging(c, job.AttachmentID) +} + +func (w *Worker) backoff(attempt int) time.Duration { + d := w.cfg.BackoffBase << min(attempt-1, 20) + if d > w.cfg.BackoffMax || d <= 0 { + d = w.cfg.BackoffMax + } + return d/2 + time.Duration(rand.Int64N(int64(d/2)+1)) +} + +type transient struct{ code, detail string } + +func (e *transient) Error() string { return e.code + ": " + e.detail } + +func transientErr(code, detail string) error { return &transient{code, detail} } + +func classify(err error) (string, bool) { + var p *imaging.PermanentError + var t *transient + switch { + case errors.As(err, &p): + return p.Code, true + case errors.As(err, &t): + return t.code, false + case errors.Is(err, context.DeadlineExceeded): + return "processing_timeout", false + } + return "processing_failed", false +} + +func safeDetail(err error) string { + s := err.Error() + if i := strings.Index(s, "http"); i >= 0 { + s = s[:i] + "[redacted]" + } + if len(s) > 300 { + s = s[:300] + } + return s +} + +func (w *Worker) sweeper(ctx context.Context) { + t := time.NewTicker(w.cfg.SweepEvery) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + } + c, cancel := context.WithTimeout(ctx, 30*time.Second) + if n, err := w.st.RecoverExpiredLeases(c, 100); err == nil && n > 0 { + w.log.Warn("expired leases recovered", "count", n) + } + refs, err := w.st.StaleStaging(c, w.cfg.PendingGrace, w.cfg.StagingRetain, 100) + if err == nil { + for _, r := range refs { + if err := w.s3.Remove(c, r.StagingKey); err != nil && !objstore.IsNotFound(err) { + continue + } + if err := w.st.ForgetStaging(c, r); err == nil { + w.log.Info("staging cleaned", "attachment_id", r.AttachmentID, "status", r.Status) + } + } + } + if n, err := w.st.PurgeOutbox(c, 24*time.Hour); err == nil && n > 0 { + w.log.Info("published outbox rows purged", "count", n) + } + cancel() + } +} diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 00000000..b264f91a Binary files /dev/null and b/student-tests/fixtures/sample.jpg differ diff --git a/student-tests/imagegen.py b/student-tests/imagegen.py new file mode 100644 index 00000000..d302a92a --- /dev/null +++ b/student-tests/imagegen.py @@ -0,0 +1,34 @@ +"""Synthetic PNG builders for negative/limit tests (stdlib only, nothing stored in Git).""" +import os +import struct +import zlib + + +def chunk(kind, data): + body = kind + data + return struct.pack('>I', len(data)) + body + struct.pack('>I', zlib.crc32(body) & 0xffffffff) + + +def png(width, height, raw=None, level=6, color_type=2): + """RGB (color_type=2) PNG; raw is the filtered scanline stream (default: random noise).""" + if raw is None: + row = 1 + 3 * width + raw = b''.join(b'\x00' + os.urandom(row - 1) for _ in range(height)) + ihdr = struct.pack('>IIBBBBB', width, height, 8, color_type, 0, 0, 0) + return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) + + chunk(b'IDAT', zlib.compress(raw, level)) + chunk(b'IEND', b'')) + + +def near_limit_png(limit=10 * 1024 * 1024): + """Valid noise PNG a little below the byte limit (stored deflate blocks).""" + width = 1000 + height = (limit - 64 * 1024) // (1 + 3 * width) + data = png(width, height, level=0) + assert len(data) <= limit, len(data) + return data + + +def bomb_png(width=20000, height=20000): + """Header declares 400M pixels, the compressed payload is tiny.""" + return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + + chunk(b'IDAT', zlib.compress(b'\x00' * 4096, 9)) + chunk(b'IEND', b''))