diff --git a/.env.example b/.env.example index 119384b5..f6ece76a 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,16 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +API_REPLICAS=2 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url +CLEANUP_GRACE_SECONDS=600 +CLEANUP_INTERVAL_SECONDS=300 WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_BACKOFF_BASE_MS=1000 +JOB_BACKOFF_MAX_MS=30000 STAGING_RETENTION_SECONDS=86400 diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 00000000..bf7cd075 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,91 @@ +# Изменения lab5 относительно lab4 + +## API + +- `POST /chats/{chat_id}/attachments` возвращает `202` со статусом `queued` вместо `201 ready`. Проверки при приёме те же, что в lab4. +- Оригинал сначала сохраняется в приватный S3-prefix `staging/`, а не сразу в конечный `attachments/`. +- `GET /attachments/{id}` возвращает статусы `queued`, `processing`, `ready`, `failed`, а также `error_code` и `variants`. +- `GET /attachments/{id}/content`: + - принимает `variant=original|thumbnail|crop|optimized`, неизвестное имя → 400; + - для не-ready вложения → 409; + - отсутствующий вариант ready-вложения → 404. +- `POST /chats/{chat_id}/messages`: прикрепить `failed`-вложение нельзя (409), `queued`/`processing` можно. + +## База данных + +Новая миграция `migrations/0004_jobs.sql`: + +- В `attachments` добавлены `status`, `error_code`, `staging_key`, `updated_at`. `object_key` теперь заполняет worker. +- `attachment_variants`: варианты с первичным ключом `(attachment_id, pipeline_version, name)`. +- `jobs`: задача обработки (статус, попытка, лимит попыток, время следующей попытки, lease, последняя ошибка). +- `outbox`: задания на публикацию в брокер. +- `job_replays`: аудит операторских повторов. + +## Новые компоненты + +- `broker`: Valkey Streams, отдельный сервис с volume, `appendfsync always`, `noeviction`, паролем из secrets. Stream `attachments.jobs`, группа `workers`. +- `worker`: отдельный процесс (`messenger worker`), 2 потребителя, prefetch 1, лимит 768 MiB. +- `broker.go`: формат сообщения по `jobs.schema.json`, публикация, outbox relay (работает в каждой API-реплике). +- `worker.go`: + - захват задачи с lease и fencing по `lease_owner` и `attempt`; + - обработка и фиксация результата; + - retry с backoff и jitter; + - reaper для задач с истёкшим lease; + - повторное чтение своих pending и `XAUTOCLAIM`; + - команды `failed` и `replay`. +- `imaging.go`: + - thumbnail JPEG 256, crop JPEG 128×128, optimized WebP quality 80 (вписан в 1280); + - применение EXIF orientation, alpha на белом; + - предварительное уменьшение больших изображений для экономии памяти. + +## Изменения существующего кода + +- `attachments.go`: приём в staging, транзакция `attachments` + `jobs` + `outbox`, ответ 202, загрузка вариантов, скачивание по `variant`. Janitor дополнительно чистит `staging/`. +- `handlers.go`: проверка failed-вложений при создании сообщения, точка внедрения отказа. +- `main.go`: подкоманды `worker`, `failed`, `replay`; запуск relay в API; новые переменные окружения. +- `s3.go`: listing по произвольному prefix. +- `Dockerfile`: сборка с тегом `nodynamic` (WebP без системных библиотек). +- `go.mod`: добавлены `golang.org/x/image`, `github.com/gen2brain/webp`, `github.com/rwcarlsen/goexif`. + +## Инфраструктура и скрипты + +- `compose.yaml`: сервисы `broker` и `worker`, volume `broker_data`, secrets `broker_conf` и `broker_url`, образ `mtusi-messenger:lab5`, переменные для точек внедрения отказа (по умолчанию пусты). +- `scripts/mksecrets.sh`: генерирует пароль брокера, `broker.conf` и `broker_url`. +- `scripts/restart.sh`: перезапускает также `broker` и `worker`. +- `scripts/rolling-update.sh`: обновляет также `worker`. +- `.env.example`: `WORKER_CONCURRENCY`, `JOB_MAX_ATTEMPTS`, `JOB_TIMEOUT_SECONDS`, `JOB_BACKOFF_BASE_MS`, `JOB_BACKOFF_MAX_MS`, `STAGING_RETENTION_SECONDS`. + +## Надёжность + +- Работа не теряется после ответа 202: оригинал в S3, задача и outbox в одной транзакции PostgreSQL. +- Разрыв COMMIT/publish закрыт transactional outbox. +- Повторная доставка и повторы не создают второго набора вариантов: детерминированные S3-ключи и upsert по ключу эффекта. +- Зависшие задачи возвращаются в очередь по истечении lease. +- Постоянные ошибки (`invalid_image`, `processing_failed`) уходят в failed сразу, временные повторяются до `retry_exhausted`. +- Failed-задачи можно повторить через `replay`. Повтор уже готовой задачи ничего не делает. +- Повреждение внутри PNG, которое lab4 пропускал (проверялись только заголовок и конец), теперь ловит worker при полном декодировании. + +## Тесты + +Новые: +- `tests/stack.py`: общие хелперы. +- `tests/variants.py`: размеры, форматы, sha256, без увеличения, alpha, EXIF. +- `tests/queue_recovery.py`: сценарий на 3 и повторная доставка. +- `tests/faults.py`: 5 сценариев сбоев на 4. +- `tests/worker_load.py`: конкуренция, память и timeout worker. + +Адаптированы под 202: `tests/attachments.py`, `tests/s3_checks.py` (плюс проверка очистки staging), `tests/upload_memory.py`. + +Без изменений: `tests/idempotency.py`, `tests/scenario.py`, `tests/image_gen.py`. + +## Отчёт и доказательства + +- `REPORT.md` переписан под lab5. +- Новые файлы в `evidence/`: `variants.txt`, `queue-recovery.txt`, `faults.txt`, `worker-load.json`. +- Остальные файлы evidence получены заново на стенде lab5. + +## Исправления после ревью + +- Авторизация проверяет сессию только в PostgreSQL одним запросом (`sessions` ⋈ `users`, `expires_at > now()`); запись кэша не принимается за доказательство действительности токена. Код кэша сессий (`cache.go`) удалён. +- Добавлен `tests/session_revocation.py`: Valkey остановлен → logout 204 → Valkey запущен → прежний токен 401. +- Тесты `image_gen.py`, `idempotency.py`, `s3_checks.py`, `scenario.py`, `upload_memory.py` были не добавлены в git, из-за чего `attachments.py` и `worker_load.py` падали с `ModuleNotFoundError: image_gen`. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..94f0316e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM golang:1.26-alpine@sha256:c95332c2af86b6d89b91bd0500f4b9529ccbd090a0d1855c6d1ceaa142ae8615 AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY *.go ./ +COPY migrations ./migrations +RUN CGO_ENABLED=0 GOOS=linux go build -tags nodynamic -trimpath -ldflags="-s -w" -o /out/messenger . + +FROM alpine:3.21@sha256:ce64758a109eb420d874a118f87920e625e12d3634e03b4a5573fd9f6e5d3507 +RUN adduser -D -u 10001 app +WORKDIR /app +COPY --from=build /out/messenger /app/messenger +USER 10001 +EXPOSE 8080 +ENV APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +STOPSIGNAL SIGTERM +HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \ + CMD wget -qO- http://127.0.0.1:8080/health/ready >/dev/null || exit 1 +ENTRYPOINT ["/app/messenger"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..bc4e3983 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,177 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Березин Герман / БВТ2304 +- Целевая оценка: 4 +- Commit/PR: после публикации +- Использованные источники, библиотеки и помощь ИИ: Go 1.26, pgx, go-redis (клиент Valkey Streams), minio-go v7, golang.org/x/image/draw (ресемплинг), github.com/gen2brain/webp (libwebp, транспилированный в Go), github.com/rwcarlsen/goexif (EXIF orientation); PostgreSQL 17, Valkey 8.1, SeaweedFS, HAProxy 3.0; документация RabbitMQ reliability и Redis Streams из README; код lab4 перенесён за основу; ИИ-ассистент использован при разработке ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker 29 с Compose v2 и Python 3.9+. Для `tests/variants.py` нужен Pillow (проверяет реальные пиксели JPEG/WebP). + +```sh +cp .env.example .env +sh scripts/mksecrets.sh +docker compose --env-file .env up -d --build --wait +make check && make test +``` + +Порядок старта: postgres, valkey, broker, s3 → `migrate` → `s3-init` → `api` × 2, `worker`, `janitor` → `lb`. Публичный адрес: `http://localhost:8080`. После первого старта S3 запись становится доступна через 10–30 s. + +Остановка без потери данных: `docker compose --env-file .env stop`. Намеренное удаление учебных данных: `docker compose --env-file .env down -v`. + +Операторские команды: + +```sh +docker compose --env-file .env stop worker +docker compose --env-file .env up -d --no-deps worker +docker compose --env-file .env run --rm --no-deps worker failed +docker compose --env-file .env run --rm --no-deps worker replay +``` + +Проверки (стенд запущен): + +```sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/variants.py +python3 tests/queue_recovery.py +python3 tests/faults.py +python3 tests/worker_load.py --output evidence/worker-load.json +python3 tests/attachments.py +python3 tests/s3_checks.py +python3 tests/upload_memory.py --output evidence/upload-memory.json +python3 tests/idempotency.py +python3 tests/session_revocation.py +python3 tests/scenario.py --mode failover-each --output evidence/failover-each.json +python3 tests/scenario.py --mode rolling --output evidence/rolling.json +``` + +После `make test-failover` реплику возвращают командой `docker compose --env-file .env up -d --wait api`. `faults.py`, `queue_recovery.py`, `worker_load.py` сами останавливают и пересоздают worker/api/broker/s3 и возвращают их в исходное состояние. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +клиент → HAProxy → api ×2 ──(1) S3 staging/ + └──(2) PostgreSQL: attachments(queued) + jobs + outbox — одна транзакция → 202 +api: outbox relay ──XADD──→ broker (Valkey Stream attachments.jobs, group workers) +worker ──XREADGROUP──→ claim (lease) → S3 attachments//... → COMMIT ready → XACK +``` + +| Роль | Сервис | Образ | Готовность | Volume | Сеть | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| api ×2 | `api` | собирается из исходников (`mtusi-messenger:lab5`) | `/health/ready` (PostgreSQL) | нет | backend, storage | secrets: database_url, cache_url, broker_url, S3-ключи API | +| worker | `worker` | тот же образ, команда `worker` | нет HTTP, healthcheck отключён | нет | backend, storage | database_url, broker_url, S3-ключи API | +| postgres | `postgres` | postgres:17-alpine@sha256:b0f9560a… | `pg_isready` | pg_data | backend | postgres_password | +| broker | `broker` | valkey/valkey:8.1-alpine@sha256:081c2f5c… | `valkey-cli ping` с паролем | broker_data | backend | broker.conf (`requirepass`) | +| s3 | `s3` | chrislusf/seaweedfs@sha256:c2658820… | `/dir/assign` выдаёт fid | s3_data | storage | s3_config.json | +| load_balancer | `lb` | haproxy:3.0-alpine@sha256:cf3af0fa… | проверки `/health/ready` бэкендов | нет | backend, 127.0.0.1:8080 | нет | +| Valkey (сессии проверяются только в PostgreSQL) | `valkey` | valkey/valkey:8.1-alpine | `valkey-cli ping` | valkey_data | backend | нет | + +Bootstrap: `migrate` применяет `migrations/*.sql` под advisory lock, повторный запуск пропускает применённые. `s3-init` создаёт bucket, если его нет. Consumer group `workers` создают API (перед первой публикацией) и worker при старте (`XGROUP CREATE … 0 MKSTREAM`, ошибка BUSYGROUP игнорируется). + +- **Брокер.** Valkey Streams в отдельном сервисе `broker` с `appendonly yes`, `appendfsync always`, `noeviction`. Доставка at least once: сообщение остаётся в pending (PEL) до `XACK`. Publisher confirm: ответ на `XADD` приходит после записи в AOF с fsync. Consumer ack: `XACK` после COMMIT результата в PostgreSQL. В сообщении только поля `jobs.schema.json` (поле `job` — JSON с `schema_version`, `job_id`, `type`, `attachment_id`, `pipeline_version`, `attempt`, `created_at`), без bytes, URL и credentials. +- **Что сохранено к 202.** Полный проверенный оригинал лежит в приватном S3-prefix `staging/` (тот же bucket, volume s3_data). Одной транзакцией сохранены строки `attachments` (status `queued`, staging_key, sha256, размер), `jobs` (attempt 1) и `outbox`. Проверки на приёме те же, что в lab4: 413/415/400/404, хвост и заголовок изображения, не более 40 млн пикселей. +- **Transactional outbox.** В каждой API-реплике работает relay: `select … from outbox where published_at is null and available_at <= now() for update skip locked`, затем `XADD` и `published_at = now()` в той же транзакции. Падение между COMMIT и publish оставляет неопубликованную строку, её публикует любая живая реплика. Падение после `XADD` до COMMIT пометки даёт повторную публикацию: это дубль, worker обработает его как no-op. +- **Worker.** Отдельный процесс (`messenger worker`), `WORKER_CONCURRENCY=2` потребителя, у каждого `COUNT 1`: prefetch 1, не больше двух задач одновременно. При старте потребитель дочитывает свои pending-сообщения (`XREADGROUP … 0`), каждые 15 s забирает зависшие чужие (`XAUTOCLAIM` с min-idle = lease). +- **Ключ идемпотентности.** Логическая работа — `jobs.id` (job_id), он один на вложение и не меняется при redelivery. Ключ эффекта — `(attachment_id, pipeline_version, name)`: первичный ключ `attachment_variants`, вставка через `on conflict do update`. S3-ключи детерминированы: `attachments//original`, `attachments//v1/{thumbnail.jpg,crop.jpg,optimized.webp}`. Повтор перезаписывает те же объекты и не создаёт второго набора. +- **Захват и lease.** `update jobs set status='processing', lease_owner, lease_until = now() + lease where status='queued' and next_attempt_at <= now()`. Сообщение для задачи в любом другом состоянии подтверждается без работы (`job delivery skipped`), поэтому повторная доставка ready-задачи — no-op. Lease = `JOB_TIMEOUT_SECONDS` + 15 s. Фиксация результата и перевод в retry/failed выполняются только при совпадении `lease_owner` и `attempt`. Worker, потерявший lease, не перезапишет состояние (`lease lost, result discarded`). +- **Зависший или долгий.** Задача ограничена контекстом `JOB_TIMEOUT_SECONDS` (60 s), поэтому живой worker не может законно работать дольше lease. Каждые 5 s reaper в worker переводит задачи с истёкшим lease в `queued` со следующей попыткой или в `failed`. Новый захват получает новый `attempt`. Старый процесс не сможет зафиксировать результат из-за fencing. Запись тех же детерминированных объектов безвредна. +- **Retry.** Временные ошибки (S3, PostgreSQL, timeout, истёкший lease) дают `queued` с `attempt+1`, новой строкой outbox с `available_at = now() + delay` и записью причины в `jobs.last_error`. Задержка `d = min(JOB_BACKOFF_BASE_MS·2^(n−1), JOB_BACKOFF_MAX_MS)` выбирается случайно в `[d/2, d]` (equal jitter). Всего `JOB_MAX_ATTEMPTS = 5` попыток, затем `failed` с `retry_exhausted`. Постоянные ошибки сразу дают `failed` без повторов: `invalid_image` (файл не декодируется, превышен лимит пикселей), `processing_failed` (staging отсутствует или не совпадает sha256/размер). +- **Реестр failed и replay.** Реестр — `jobs` со статусом `failed` (`worker failed` выводит attachment_id, attempt, error_code, last_error). `worker replay ` переводит в `queued` только задачу в `failed`: даёт новый бюджет из `JOB_MAX_ATTEMPTS` попыток, пишет аудит в `job_replays` и строку outbox. Для `ready`/`queued`/`processing` результат `skipped_`, дубля нет. +- **Варианты.** Все варианты строятся из пикселей и кодируются заново, поэтому EXIF и прочие метаданные не копируются. EXIF orientation (JPEG) применяется, alpha компонуется на белый, ресемплинг Catmull-Rom. Изображения больше 2560 px сначала уменьшаются вдвое блоком 2×2, чтобы ограничить промежуточный буфер. Итоговые размеры считаются от исходных. + - `thumbnail`: JPEG q=85, вписан в 256×256, без увеличения, размеры округляются вниз, минимум 1 px; fixture 320×200 → 256×160. + - `crop`: JPEG q=85, центральный квадрат, 128×128, маленькие исходники увеличиваются. + - `optimized`: WebP lossy, quality 80, method 4, вписан в 1280×1280 без увеличения. Уменьшение каждого маленького файла не обещается. +- **Состояния API.** POST upload → 202 `queued`. GET метаданных возвращает `queued` / `processing` / `ready` / `failed`. Для `ready` в `variants` три записи с размерами и sha256 реального файла. `/content` для не-ready → 409. `variant=` с неизвестным именем → 400. Отсутствующий вариант ready-объекта → 404. Прикрепить `failed`-вложение к сообщению → 409, `queued`/`processing` можно. Ready ставится одной транзакцией после записи оригинала и всех трёх вариантов. +- **Staging.** Удаляется worker'ом после COMMIT ready. Если удаление не прошло, `janitor` удаляет staging ready-вложений и staging без строки (старше `CLEANUP_GRACE_SECONDS`). Staging `failed` хранится `STAGING_RETENTION_SECONDS` (24 ч) ради replay. Staging `queued`/`processing` не удаляется. +- **Критичные зависимости.** `/health/ready` проверяет только PostgreSQL. Брокер не нужен синхронному пути: при его недоступности загрузки принимаются, задания копятся в outbox. Upload и download требуют S3 (иначе 503). +- **Внедрение отказов.** `FAULT_POINT` по умолчанию пуст. В тестах через `API_FAULT_POINT=after_commit` (выход процесса после COMMIT, до ответа и publish) и `WORKER_FAULT_POINT=after_store|before_ack` (выход после записи в S3 до COMMIT / после COMMIT до ack). +- Перенесено из lab4 без изменения смысла: сессии, идемпотентность сообщений, HAProxy, приватный bucket и отдельные ключи, janitor (дополнен очисткой staging), лимиты загрузки. Изменено: upload 201 → 202, `attachments.object_key` заполняется worker'ом, добавлена миграция `0004_jobs.sql`. + +### Окна отказа + +| Окно | Что происходит | Восстановление | +| --- | --- | --- | +| после staging до записи задания | объект в `staging/` без строки, клиент получает 503 или обрыв | janitor удаляет его после grace period | +| после COMMIT до publish | строка outbox не опубликована | relay любой API-реплики публикует; проверено (`faults.py`, 1) | +| после publish до confirm/пометки | сообщение в stream, строка outbox не помечена | повторная публикация → дубль → no-op при захвате | +| после записи S3 до COMMIT результата | задача `processing`, объекты записаны | lease истекает → reaper → попытка 2 перезаписывает те же ключи; проверено (`faults.py`, 2) | +| после COMMIT результата до ack | ready, сообщение в PEL | после рестарта worker дочитывает pending → no-op → ack; проверено (`faults.py`, 3) | + +### Соответствие .env.example + +| Переменная | Где используется | +| --- | --- | +| `API_REPLICAS` | `deploy.replicas` сервиса api | +| `S3_REGION`, `S3_BUCKET` | api, worker, janitor, s3-init, генерация прав в `mksecrets.sh` | +| `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` | api (приём) и worker (повторная проверка) | +| `WORKER_CONCURRENCY` | число потребителей worker | +| `JOB_MAX_ATTEMPTS` | api записывает в `jobs.max_attempts`, replay выдаёт новый бюджет | +| `JOB_TIMEOUT_SECONDS` | таймаут задачи, lease = значение + 15 s | +| `JOB_BACKOFF_BASE_MS`, `JOB_BACKOFF_MAX_MS` | backoff повторов | +| `STAGING_RETENTION_SECONDS` | хранение staging failed-вложений (janitor) | +| `CLEANUP_GRACE_SECONDS`, `CLEANUP_INTERVAL_SECONDS` | janitor | +| `BROKER_URL_FILE`, `DATABASE_URL_FILE`, `S3_*_FILE` | задаются в compose как пути к secrets | + +### Лимиты worker + +| Ограничение | Значение | +| --- | --- | +| Bytes | размер staging должен совпасть с метаданными; чтение не более `MAX_UPLOAD_BYTES`, sha256 сверяется | +| Пиксели | `image.DecodeConfig` до декодирования, не более `MAX_IMAGE_PIXELS` (40 млн), иначе `invalid_image` | +| Время | `JOB_TIMEOUT_SECONDS` на задачу (S3, БД, проверки между этапами) | +| Память | 2 задачи одновременно, `GOMEMLIMIT=600MiB`, лимит контейнера 768 MiB, CPU 1.0 | +| Конкуренция | `WORKER_CONCURRENCY=2`, prefetch (`COUNT`) 1 на потребителя | ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke (202 → ready, оригинал, настоящий JPEG 256×160) | `make check && make test` | PASS | evidence/smoke.txt | +| Отозванная сессия не воскресает после перезапуска Valkey | `tests/session_revocation.py` | Valkey остановлен → logout 204 → Valkey запущен → прежний токен 401 | evidence/session-revocation.txt | +| Минимум lab4 сохранён (валидация, доступ, приватный bucket, отказ S3, идемпотентность) | `tests/attachments.py`, `tests/s3_checks.py`, `tests/idempotency.py` | PASS: 413/415/400/404, анонимный доступ 403, 503 при остановленном S3 | evidence/attachments.txt, evidence/s3-checks.txt, evidence/idempotency.txt | +| Restart всех stores с volumes | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; перезапущены postgres, valkey, broker, s3, api, lb, janitor, worker (состояния до/после в файле) | evidence/persistence.txt | +| Failover каждой API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS для api-1 и api-2, 0–1 неуспешная попытка, восстановление 3.5 s | evidence/failover-1.txt, evidence/failover-2.txt | +| Worker остановлен → upload 202 → сообщение → content 409 → рестарт broker и API → worker запущен → ready | `tests/queue_recovery.py` | задача есть в stream до и после рестарта broker, статус `queued` без worker, после старта ready, attempt 1, 3 варианта | evidence/queue-recovery.txt | +| Повторная доставка не размножает варианты | `tests/queue_recovery.py` (ручной `XADD` дубля) | `job delivery skipped`, ack, `updated_at` не изменился, 3 варианта | evidence/queue-recovery.txt | +| Thumbnail, crop 128×128, optimized WebP, качество/метаданные | `tests/variants.py` | размеры и форматы по реальным байтам, sha256 совпадает, без увеличения, alpha → белый, EXIF orientation 6 применена, EXIF в вариантах нет | evidence/variants.txt | +| Crash между COMMIT и publish | `tests/faults.py`, сценарий 1 | API упал после COMMIT (202 не получен), outbox опубликован другой репликой, ready attempt 1 | evidence/faults.txt | +| Остановка worker после записи объекта до COMMIT, timeout/lease | `tests/faults.py`, сценарий 2 | lease истёк, reaper вернул в очередь, ready attempt 2, вариантов 3 | evidence/faults.txt | +| Остановка worker после COMMIT до ack | `tests/faults.py`, сценарий 3 | после рестарта pending доставлен повторно, no-op, attempt 1, PEL пуст | evidence/faults.txt | +| Backoff с jitter, ограниченный retry, failed, replay | `tests/faults.py`, сценарий 4 | S3 остановлен: 4 повтора с задержками 931/1284/2760/7484 мс (окна [d/2, d]), `retry_exhausted`, задача в реестре, content и прикрепление → 409; replay → ready, повторный replay → `skipped_ready`, аудит 1 строка | evidence/faults.txt | +| Постоянная ошибка без повторов | `tests/faults.py`, сценарий 5 | PNG с повреждённым IDAT (проходит проверку API) → `invalid_image` на попытке 1 | evidence/faults.txt | +| Ограничение конкуренции, памяти и времени worker | `tests/worker_load.py` | 6 изображений по 39.9 млн пикселей + 20 малых: одновременно не более 2 processing, пик RSS 617 MiB при лимите 768 MiB, без OOM; при `JOB_TIMEOUT_SECONDS=1` — `context deadline exceeded`, 5 попыток, `retry_exhausted` | evidence/worker-load.json | +| Очистка staging после безопасной фиксации | `tests/s3_checks.py` | staging ready-вложения удалён, staging queued-задачи сохранён janitor'ом, staging без строки удалён | evidence/s3-checks.txt | +| Память API при загрузках | `tests/upload_memory.py` | 12 потоков × 3 файла по 9.3 MiB, 36/36 → 202, пик api 119 / 100 MiB при лимите 192 | evidence/upload-memory.json | +| Отказ реплики и rolling update под нагрузкой | `tests/scenario.py` | см. эксперименты | evidence/failover-each.json, evidence/rolling.json | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Стенд: один хост, Docker 29.9.2, Compose 5.6.1, лимиты контейнеров как выше. Нагрузка сообщений — 8 потоков, closed-loop, повтор с тем же ключом. Нагрузка worker — 6 PNG 6320×6320 (116 545 байт, однотонные: проверка лимита пикселей, а не сети) и 20 копий fixture. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Worker остановлен, upload | 202, работа ждёт | `queued`, content 409, текст работает | после старта worker ready, потерь нет | +| Restart broker и API с volumes при остановленном worker | задача не теряется | job_id в stream до и после | потерь нет | +| Падение API после COMMIT | задача не теряется без ручного вмешательства | outbox опубликован живой репликой | ready в пределах секунд, потерь нет | +| Падение worker после записи S3 | повтор того же результата | ready с attempt 2 после истечения lease 25 s (тест с `JOB_TIMEOUT_SECONDS=10`) | ≈ lease + 5 s, дублей нет | +| Падение worker после COMMIT | повторная доставка = no-op | skipped, ack | сразу после рестарта | +| S3 недоступен для worker | ограниченные повторы, failed | 5 попыток за ≈ 13 s, `retry_exhausted`, replay → ready | после replay, потерь нет | +| 6×40 Мп + 20 малых | не более 2 задач одновременно, память в лимите | все ready за 13.1 s, пик RSS 617 MiB / 768 | без OOM и рестартов | +| Остановка каждой API-реплики под нагрузкой | запись продолжается | 2 и 0 неуспешных попыток, RPS 439 / 447 против 524 до отказа | возврат реплики около 6.5 s | +| Rolling update | без простоя | 4 неуспешные попытки, RPS 615 | старые реплики не отвечают | + +Первая версия без предварительного уменьшения упиралась в лимит 512 MiB (RSS 780 MiB при двух 40-Мп задачах). Причина — промежуточный буфер Catmull-Rom размером ширина_цели × высота_источника × 32 байта. После уменьшения блоком 2×2 пик стал около 600 MiB, а время двух таких задач сократилось с 11 до 4.6 s. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Декодирование и ресемплинг одного этапа нельзя прервать: timeout проверяется между этапами и на S3/БД-операциях. Длительность этапа ограничена лимитом пикселей (около 2 s на 40 Мп). +- Broker, PostgreSQL, S3 и HAProxy — единичные экземпляры. `appendfsync always` надёжнее, но медленнее everysec. Stream не обрезается (`XTRIM` не реализован), он растёт на одну запись на попытку. +- Relay работает только в API: если остановлены обе реплики API, принятые задания ждут в outbox. Задержка relay до 300 ms. +- Имена потребителей привязаны к hostname контейнера: после пересоздания worker старые pending переходят к новым через `XAUTOCLAIM` только после lease (75 s), а старые consumer'ы остаются в группе. +- EXIF orientation читается только у JPEG; eXIf-чанк PNG не учитывается. +- `pipeline_version` фиксирован (1). Версионирование нескольких pipeline (трек 5) не делалось. +- Replay — операторская CLI-команда без HTTP API. `last_error` хранит внутренний текст ошибки только в БД и наружу не выдаётся. +- WebP кодируется транспилированным libwebp (без CGo): медленнее нативного, но без системных библиотек. +- Секреты лежат в `./secrets` вне Git. Для чтения процессом с UID 10001 у них права 644: это допустимо только для локального стенда. diff --git a/attachments.go b/attachments.go new file mode 100644 index 00000000..b8894990 --- /dev/null +++ b/attachments.go @@ -0,0 +1,530 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "hash" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "log/slog" + "net/http" + "os" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + objectPrefix = "attachments/" + stagingPrefix = "staging/" + probeHead = 256 << 10 + tailLen = 12 +) + +var ( + pngMagic = []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'} + pngTail = []byte{0, 0, 0, 0, 'I', 'E', 'N', 'D', 0xae, 0x42, 0x60, 0x82} + jpegMagic = []byte{0xff, 0xd8, 0xff} + jpegTail = []byte{0xff, 0xd9} +) + +type limits struct { + maxBytes int64 + maxPixels int64 + uploadSlots int + maxAttempts int +} + +type attachment struct { + ID, ChatID, OwnerID, Filename, ContentType, SHA256, Key, Status string + ErrorCode *string + Size int64 + Created, Updated time.Time + Variants []variant +} + +type variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + Size int64 `json:"size_bytes"` + Width int `json:"width"` + Height int `json:"height"` + SHA256 string `json:"sha256"` + key string +} + +func (x *attachment) json() map[string]any { + variants := x.Variants + if variants == nil { + variants = []variant{} + } + return map[string]any{ + "id": x.ID, + "chat_id": x.ChatID, + "owner_id": x.OwnerID, + "filename": x.Filename, + "content_type": x.ContentType, + "size_bytes": x.Size, + "sha256": x.SHA256, + "status": x.Status, + "variants": variants, + "error_code": x.ErrorCode, + "created_at": formatTime(x.Created), + "updated_at": formatTime(x.Updated), + } +} + +type probe struct { + sum hash.Hash + size int64 + head []byte + tail []byte +} + +func newProbe() *probe { + return &probe{sum: sha256.New()} +} + +func (p *probe) Write(b []byte) (int, error) { + p.sum.Write(b) + p.size += int64(len(b)) + if room := probeHead - len(p.head); room > 0 { + if room > len(b) { + room = len(b) + } + p.head = append(p.head, b[:room]...) + } + p.tail = append(p.tail, b...) + if len(p.tail) > tailLen { + p.tail = append(p.tail[:0], p.tail[len(p.tail)-tailLen:]...) + } + return len(b), nil +} + +func (p *probe) digest() string { + return hex.EncodeToString(p.sum.Sum(nil)) +} + +var ( + errNotImage = errors.New("not an image") + errTooBig = errors.New("too large") +) + +func (p *probe) verify(contentType string, maxPixels int64) error { + cfg, format, err := image.DecodeConfig(bytes.NewReader(p.head)) + if err != nil { + return errNotImage + } + var tail []byte + switch contentType { + case "image/png": + if format != "png" { + return errNotImage + } + tail = pngTail + case "image/jpeg": + if format != "jpeg" { + return errNotImage + } + tail = jpegTail + } + if !bytes.HasSuffix(p.tail, tail) { + return errNotImage + } + if int64(cfg.Width)*int64(cfg.Height) > maxPixels { + return errTooBig + } + return nil +} + +func magicMatches(contentType string, head []byte) bool { + switch contentType { + case "image/png": + return bytes.HasPrefix(head, pngMagic) + case "image/jpeg": + return bytes.HasPrefix(head, jpegMagic) + } + return false +} + +func validFilename(name string) bool { + n := utf8.RuneCountInString(name) + if n < 1 || n > 255 || !utf8.ValidString(name) { + return false + } + for _, r := range name { + if r == '/' || r == '\\' || r == 0 || unicode.IsControl(r) { + return false + } + } + return true +} + +type trackedReader struct { + r io.Reader + err error +} + +func (t *trackedReader) Read(b []byte) (int, error) { + n, err := t.r.Read(b) + if err != nil && err != io.EOF && t.err == nil { + t.err = err + } + return n, err +} + +func isTooLarge(err error) bool { + var mbe *http.MaxBytesError + return errors.As(err, &mbe) +} + +func (a *App) dropObject(key string) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := a.store.Remove(ctx, key); err != nil { + a.log.Warn("object cleanup failed", "key", key, "err", err.Error()) + } +} + +func (a *App) handleUploadAttachment(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + chatID := r.PathValue("chat_id") + if !isUUID(chatID) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id") + return + } + ok, err := a.isMember(r.Context(), chatID, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if !ok { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + select { + case a.uploads <- struct{}{}: + defer func() { <-a.uploads }() + default: + w.Header().Set("Retry-After", "1") + a.writeErr(w, reqID, 429, "rate_limited", "too many concurrent uploads") + return + } + filename := r.URL.Query().Get("filename") + if !validFilename(filename) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid filename") + return + } + contentType := strings.ToLower(strings.TrimSpace(strings.Split(r.Header.Get("Content-Type"), ";")[0])) + if contentType != "image/png" && contentType != "image/jpeg" { + a.writeErr(w, reqID, 415, "unsupported_media_type", "Content-Type must be image/png or image/jpeg") + return + } + if r.ContentLength > a.limits.maxBytes { + a.writeErr(w, reqID, 413, "payload_too_large", "file too large") + return + } + body := &trackedReader{r: http.MaxBytesReader(w, r.Body, a.limits.maxBytes)} + head := make([]byte, 16) + n, _ := io.ReadFull(body, head) + if body.err != nil { + if isTooLarge(body.err) { + a.writeErr(w, reqID, 413, "payload_too_large", "file too large") + } else { + a.writeErr(w, reqID, 400, "invalid_request", "upload interrupted") + } + return + } + head = head[:n] + if !magicMatches(contentType, head) { + a.writeErr(w, reqID, 415, "unsupported_media_type", "content does not match image type") + return + } + + id := newUUID() + key := stagingPrefix + id + pr := newProbe() + size := r.ContentLength + if size < 0 { + size = -1 + } + src := io.TeeReader(io.MultiReader(bytes.NewReader(head), body), pr) + putCtx, cancel := context.WithTimeout(r.Context(), 60*time.Second) + defer cancel() + if err := a.store.Put(putCtx, key, src, size, contentType); err != nil { + a.dropObject(key) + switch { + case isTooLarge(body.err): + a.writeErr(w, reqID, 413, "payload_too_large", "file too large") + case body.err != nil: + a.writeErr(w, reqID, 400, "invalid_request", "upload interrupted") + default: + a.log.Error("s3 put failed", "request_id", reqID, "err", err.Error()) + a.writeErr(w, reqID, 503, "unavailable", "storage unavailable") + } + return + } + if err := pr.verify(contentType, a.limits.maxPixels); err != nil { + a.dropObject(key) + if err == errTooBig { + a.writeErr(w, reqID, 413, "payload_too_large", "image too large") + } else { + a.writeErr(w, reqID, 415, "unsupported_media_type", "invalid image") + } + return + } + + now := nowUTC() + att := &attachment{ + ID: id, ChatID: chatID, OwnerID: u.ID, Filename: filename, ContentType: contentType, + SHA256: pr.digest(), Size: pr.size, Status: "queued", Created: now, Updated: now, + } + if err := a.enqueue(r.Context(), att, key); err != nil { + a.dropObject(key) + a.log.Error("enqueue failed", "request_id", reqID, "err", err.Error()) + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if a.fault == "after_commit" { + a.log.Error("fault injected", "point", "after_commit", "attachment_id", att.ID) + os.Exit(3) + } + a.writeJSON(w, 202, att.json()) +} + +func (a *App) enqueue(ctx context.Context, att *attachment, key string) error { + ctx, cancel := dbCtx(ctx) + defer cancel() + tx, err := a.db.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + if _, err := tx.Exec(ctx, ` + insert into attachments(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key, status, created_at, updated_at) + values($1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, 'queued', $9, $9)`, + att.ID, att.ChatID, att.OwnerID, att.Filename, att.ContentType, att.Size, att.SHA256, key, att.Created); err != nil { + return err + } + jobID := newUUID() + if _, err := tx.Exec(ctx, ` + insert into jobs(id, attachment_id, pipeline_version, status, attempt, max_attempts, next_attempt_at, created_at, updated_at) + values($1::uuid, $2::uuid, $3, 'queued', 1, $4, $5, $5, $5)`, + jobID, att.ID, pipelineVersion, a.limits.maxAttempts, att.Created); err != nil { + return err + } + if _, err := tx.Exec(ctx, ` + insert into outbox(job_id, attempt, available_at, created_at) values($1::uuid, 1, $2, $2)`, jobID, att.Created); err != nil { + return err + } + return tx.Commit(ctx) +} + +func (a *App) loadAttachment(ctx context.Context, id, userID string) (*attachment, error) { + var x attachment + err := a.db.QueryRow(ctx, ` + select a.id::text, a.chat_id::text, a.owner_id::text, a.filename, a.content_type, + a.size_bytes, a.sha256, coalesce(a.object_key, ''), a.status, a.error_code, a.created_at, a.updated_at + from attachments a + join chat_members m on m.chat_id = a.chat_id and m.user_id = $2::uuid + where a.id = $1::uuid`, id, userID). + Scan(&x.ID, &x.ChatID, &x.OwnerID, &x.Filename, &x.ContentType, &x.Size, &x.SHA256, &x.Key, &x.Status, &x.ErrorCode, &x.Created, &x.Updated) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if x.Status != "ready" { + return &x, nil + } + rows, err := a.db.Query(ctx, ` + select name, content_type, size_bytes, width, height, sha256, object_key + from attachment_variants where attachment_id = $1::uuid and pipeline_version = $2 + order by array_position(array['thumbnail', 'crop', 'optimized'], name)`, id, pipelineVersion) + if err != nil { + return nil, err + } + x.Variants, err = pgx.CollectRows(rows, func(row pgx.CollectableRow) (variant, error) { + var v variant + return v, row.Scan(&v.Name, &v.ContentType, &v.Size, &v.Width, &v.Height, &v.SHA256, &v.key) + }) + if err != nil { + return nil, err + } + return &x, nil +} + +func (a *App) attachmentFor(w http.ResponseWriter, r *http.Request) *attachment { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return nil + } + id := r.PathValue("attachment_id") + if !isUUID(id) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid attachment_id") + return nil + } + ctx, cancel := dbCtx(r.Context()) + defer cancel() + att, err := a.loadAttachment(ctx, id, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return nil + } + if att == nil { + a.writeErr(w, reqID, 404, "not_found", "attachment not found") + return nil + } + return att +} + +func (a *App) handleGetAttachment(w http.ResponseWriter, r *http.Request) { + if att := a.attachmentFor(w, r); att != nil { + a.writeJSON(w, 200, att.json()) + } +} + +func (a *App) handleDownloadAttachment(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + name := r.URL.Query().Get("variant") + if name == "" { + name = "original" + } + if name != "original" && name != "thumbnail" && name != "crop" && name != "optimized" { + a.writeErr(w, reqID, 400, "invalid_request", "invalid variant") + return + } + att := a.attachmentFor(w, r) + if att == nil { + return + } + if att.Status != "ready" { + a.writeErr(w, reqID, 409, "conflict", "attachment is not ready") + return + } + key, contentType := att.Key, att.ContentType + if name != "original" { + key = "" + for _, v := range att.Variants { + if v.Name == name { + key, contentType = v.key, v.ContentType + } + } + if key == "" { + a.writeErr(w, reqID, 404, "not_found", "variant not found") + return + } + } + ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) + defer cancel() + obj, info, err := a.store.Get(ctx, key) + if err != nil { + if isNoSuchKey(err) { + a.writeErr(w, reqID, 404, "not_found", "attachment not found") + return + } + a.log.Error("s3 get failed", "request_id", reqID, "err", err.Error()) + a.writeErr(w, reqID, 503, "unavailable", "storage unavailable") + return + } + defer obj.Close() + first := make([]byte, 32<<10) + n, err := io.ReadFull(obj, first) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + a.log.Error("s3 read failed", "request_id", reqID, "err", err.Error()) + a.writeErr(w, reqID, 503, "unavailable", "storage unavailable") + return + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", strconv.FormatInt(info.Size, 10)) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(200) + _, _ = w.Write(first[:n]) + _, _ = io.Copy(w, obj) +} + +var keepQueries = map[string]string{ + objectPrefix: ` + select k from unnest($1::text[]) k + where exists(select 1 from attachments where object_key = k) + or exists(select 1 from attachment_variants where object_key = k)`, + stagingPrefix: ` + select k from unnest($1::text[]) k + where exists(select 1 from attachments where staging_key = k and (status in ('queued', 'processing') + or (status = 'failed' and updated_at > now() - make_interval(secs => $2))))`, +} + +func cleanupOrphans(ctx context.Context, db *pgxpool.Pool, store *Store, grace, retention time.Duration, log *slog.Logger) (int, error) { + cutoff := time.Now().Add(-grace) + removed := 0 + flush := func(prefix string, keys []string) error { + if len(keys) == 0 { + return nil + } + args := []any{keys} + if prefix == stagingPrefix { + args = append(args, retention.Seconds()) + } + rows, err := db.Query(ctx, keepQueries[prefix], args...) + if err != nil { + return err + } + kept, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return err + } + known := map[string]bool{} + for _, k := range kept { + known[k] = true + } + for _, k := range keys { + if known[k] { + continue + } + if err := store.Remove(ctx, k); err != nil { + return err + } + log.Info("orphan object removed", "key", k) + removed++ + } + return nil + } + for _, prefix := range []string{objectPrefix, stagingPrefix} { + var batch []string + for obj := range store.List(ctx, prefix) { + if obj.Err != nil { + return removed, obj.Err + } + if obj.LastModified.After(cutoff) { + continue + } + batch = append(batch, obj.Key) + if len(batch) == 500 { + if err := flush(prefix, batch); err != nil { + return removed, err + } + batch = batch[:0] + } + } + if err := flush(prefix, batch); err != nil { + return removed, err + } + } + return removed, nil +} diff --git a/auth.go b/auth.go new file mode 100644 index 00000000..cacc42d0 --- /dev/null +++ b/auth.go @@ -0,0 +1,131 @@ +package main + +import ( + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" + "unicode/utf8" + + "golang.org/x/crypto/argon2" +) + +const ( + argonTime = 2 + argonMemory = 19456 + argonThreads = 1 + argonKeyLen = 32 + argonSaltLen = 16 +) + +func hashPassword(password string) (string, error) { + salt := make([]byte, argonSaltLen) + if _, err := rand.Read(salt); err != nil { + return "", err + } + hash := argon2.IDKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen) + b64Salt := base64.RawStdEncoding.EncodeToString(salt) + b64Hash := base64.RawStdEncoding.EncodeToString(hash) + return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", + argon2.Version, argonMemory, argonTime, argonThreads, b64Salt, b64Hash), nil +} + +func checkPassword(encoded, password string) (bool, error) { + parts := strings.Split(encoded, "$") + if len(parts) != 6 || parts[1] != "argon2id" { + return false, errors.New("bad hash") + } + var version int + if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil { + return false, err + } + var memory, timeCost, threads int + for _, p := range strings.Split(parts[3], ",") { + kv := strings.SplitN(p, "=", 2) + if len(kv) != 2 { + return false, errors.New("bad params") + } + n, err := strconv.Atoi(kv[1]) + if err != nil { + return false, err + } + switch kv[0] { + case "m": + memory = n + case "t": + timeCost = n + case "p": + threads = n + } + } + salt, err := base64.RawStdEncoding.DecodeString(parts[4]) + if err != nil { + return false, err + } + want, err := base64.RawStdEncoding.DecodeString(parts[5]) + if err != nil { + return false, err + } + got := argon2.IDKey([]byte(password), salt, uint32(timeCost), uint32(memory), uint8(threads), uint32(len(want))) + return subtle.ConstantTimeCompare(got, want) == 1, nil +} + +func newToken() (string, []byte, string, error) { + raw := make([]byte, 32) + if _, err := rand.Read(raw); err != nil { + return "", nil, "", err + } + token := base64.RawURLEncoding.EncodeToString(raw) + sum := sha256.Sum256([]byte(token)) + hash := make([]byte, len(sum)) + copy(hash, sum[:]) + return token, hash, hex.EncodeToString(hash), nil +} + +func tokenHash(token string) ([]byte, string) { + sum := sha256.Sum256([]byte(token)) + hash := make([]byte, len(sum)) + copy(hash, sum[:]) + return hash, hex.EncodeToString(hash) +} + +func parseBasic(header string) (username, password string, ok bool) { + const prefix = "Basic " + if !strings.HasPrefix(header, prefix) { + return "", "", false + } + decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(header[len(prefix):])) + if err != nil { + return "", "", false + } + s := string(decoded) + i := strings.IndexByte(s, ':') + if i < 0 { + return "", "", false + } + username = s[:i] + password = s[i+1:] + return username, password, true +} + +func parseBearer(header string) (string, bool) { + const prefix = "Bearer " + if !strings.HasPrefix(header, prefix) { + return "", false + } + tok := strings.TrimSpace(header[len(prefix):]) + if tok == "" { + return "", false + } + return tok, true +} + +func validPassword(p string) bool { + n := utf8.RuneCountInString(p) + return n >= 12 && n <= 128 +} diff --git a/broker.go b/broker.go new file mode 100644 index 00000000..1b6cad9b --- /dev/null +++ b/broker.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "encoding/json" + "log/slog" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" +) + +const ( + jobStream = "attachments.jobs" + jobGroup = "workers" + jobType = "attachment.process.v1" + pipelineVersion = 1 +) + +type jobMessage struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt string `json:"created_at"` +} + +type Broker struct { + rdb *redis.Client +} + +func openBroker() (*Broker, error) { + raw, err := readSecret("BROKER_URL", "BROKER_URL_FILE") + if err != nil { + return nil, err + } + opt, err := redis.ParseURL(raw) + if err != nil { + return nil, err + } + opt.DialTimeout = 2 * time.Second + opt.ReadTimeout = 3 * time.Second + opt.WriteTimeout = 3 * time.Second + opt.MaxRetries = 0 + return &Broker{rdb: redis.NewClient(opt)}, nil +} + +func (b *Broker) ensureGroup(ctx context.Context) error { + err := b.rdb.XGroupCreateMkStream(ctx, jobStream, jobGroup, "0").Err() + if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") { + return err + } + return nil +} + +func (b *Broker) publish(ctx context.Context, m jobMessage) error { + body, err := json.Marshal(m) + if err != nil { + return err + } + return b.rdb.XAdd(ctx, &redis.XAddArgs{Stream: jobStream, Values: map[string]any{"job": string(body)}}).Err() +} + +func relayOnce(ctx context.Context, db *pgxpool.Pool, b *Broker) (int, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + tx, err := db.Begin(ctx) + if err != nil { + return 0, err + } + defer tx.Rollback(ctx) + rows, err := tx.Query(ctx, ` + select o.id, o.job_id::text, o.attempt, j.attachment_id::text, j.pipeline_version, j.created_at + from outbox o + join jobs j on j.id = o.job_id + where o.published_at is null and o.available_at <= now() + order by o.id + limit 50 + for update of o skip locked`) + if err != nil { + return 0, err + } + var ids []int64 + var msgs []jobMessage + for rows.Next() { + var id int64 + var created time.Time + m := jobMessage{SchemaVersion: 1, Type: jobType} + if err := rows.Scan(&id, &m.JobID, &m.Attempt, &m.AttachmentID, &m.PipelineVersion, &created); err != nil { + rows.Close() + return 0, err + } + m.CreatedAt = formatTime(created) + ids = append(ids, id) + msgs = append(msgs, m) + } + rows.Close() + if err := rows.Err(); err != nil { + return 0, err + } + if len(ids) == 0 { + return 0, nil + } + if err := b.ensureGroup(ctx); err != nil { + return 0, err + } + for _, m := range msgs { + if err := b.publish(ctx, m); err != nil { + return 0, err + } + } + if _, err := tx.Exec(ctx, `update outbox set published_at = now() where id = any($1)`, ids); err != nil { + return 0, err + } + return len(ids), tx.Commit(ctx) +} + +func runRelay(ctx context.Context, db *pgxpool.Pool, b *Broker, log *slog.Logger) { + for ctx.Err() == nil { + n, err := relayOnce(ctx, db, b) + if err != nil { + log.Warn("outbox relay failed", "err", err.Error()) + time.Sleep(2 * time.Second) + continue + } + if n > 0 { + log.Info("outbox published", "count", n) + continue + } + time.Sleep(300 * time.Millisecond) + } +} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..faeb251c --- /dev/null +++ b/compose.yaml @@ -0,0 +1,321 @@ +x-api-env: &api-env + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + SESSION_STORE: postgres + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-10} + GOMEMLIMIT: 150MiB + SHUTDOWN_TIMEOUT_SECONDS: 10 + DRAIN_SECONDS: 3 + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + BROKER_URL_FILE: /run/secrets/broker_url + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + FAULT_POINT: ${API_FAULT_POINT:-} + +services: + postgres: + image: postgres:17-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24 + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_DB: messenger + secrets: + - postgres_password + volumes: + - pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + networks: + - backend + restart: unless-stopped + + valkey: + image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: ["valkey-server", "--appendonly", "yes", "--save", "60", "1"] + volumes: + - valkey_data:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: unless-stopped + + broker: + image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: ["valkey-server", "/run/secrets/broker_conf", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory-policy", "noeviction"] + secrets: + - broker_conf + volumes: + - broker_data:/data + healthcheck: + test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$(sed -n 's/^requirepass //p' /run/secrets/broker_conf)\" ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: unless-stopped + + s3: + image: chrislusf/seaweedfs@sha256:c2658820e763294a9e9de7b36c81e59b879e3459148351aee4e31a126ca64860 + command: + - server + - -dir=/data + - -ip=s3 + - -master.volumeSizeLimitMB=128 + - -volume.max=30 + - -s3 + - -s3.port=9000 + - -s3.config=/run/secrets/s3_config + secrets: + - s3_config + volumes: + - s3_data:/data + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9333/dir/assign | grep -q fid"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + networks: + - storage + restart: unless-stopped + + s3-init: + image: mtusi-messenger:lab5 + command: ["s3-init"] + environment: + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + depends_on: + s3: + condition: service_healthy + migrate: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - storage + restart: "no" + + janitor: + image: mtusi-messenger:lab5 + command: ["janitor"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + CLEANUP_GRACE_SECONDS: ${CLEANUP_GRACE_SECONDS:-600} + CLEANUP_INTERVAL_SECONDS: ${CLEANUP_INTERVAL_SECONDS:-300} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3-init: + condition: service_completed_successfully + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.25" + memory: 64M + networks: + - backend + - storage + restart: unless-stopped + + worker: + image: mtusi-messenger:lab5 + command: ["worker"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_BACKOFF_BASE_MS: ${JOB_BACKOFF_BASE_MS:-1000} + JOB_BACKOFF_MAX_MS: ${JOB_BACKOFF_MAX_MS:-30000} + GOMEMLIMIT: 600MiB + FAULT_POINT: ${WORKER_FAULT_POINT:-} + secrets: + - database_url + - broker_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3-init: + condition: service_completed_successfully + broker: + condition: service_healthy + healthcheck: + disable: true + stop_grace_period: 30s + deploy: + resources: + limits: + cpus: "1.0" + memory: 768M + networks: + - backend + - storage + restart: unless-stopped + + migrate: + build: . + image: mtusi-messenger:lab5 + command: ["migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: "no" + + api: + image: mtusi-messenger:lab5 + environment: *api-env + secrets: + - database_url + - cache_url + - broker_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3-init: + condition: service_completed_successfully + valkey: + condition: service_healthy + broker: + condition: service_healthy + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "0.5" + memory: 192M + networks: + - backend + - storage + restart: unless-stopped + + lb: + image: haproxy:3.0-alpine@sha256:cf3af0facf7eb22b9fee463daf0fbc07ead740b8150696dce724264faf7b4318 + ports: + - "127.0.0.1:8080:8080" + volumes: + - ./lb/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + - api + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: unless-stopped + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + broker_conf: + file: ./secrets/broker.conf + broker_url: + file: ./secrets/broker_url + s3_config: + file: ./secrets/s3_config.json + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + +volumes: + pg_data: + valkey_data: + broker_data: + s3_data: + +networks: + backend: + driver: bridge + storage: + driver: bridge diff --git a/db.go b/db.go new file mode 100644 index 00000000..5b051b33 --- /dev/null +++ b/db.go @@ -0,0 +1,57 @@ +package main + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func readSecret(envKey, fileKey string) (string, error) { + if v := os.Getenv(envKey); v != "" { + return strings.TrimSpace(v), nil + } + path := os.Getenv(fileKey) + if path == "" { + return "", fmt.Errorf("%s or %s required", envKey, fileKey) + } + b, err := os.ReadFile(path) + if err != nil { + return "", err + } + return strings.TrimSpace(string(b)), nil +} + +func openDB(ctx context.Context) (*pgxpool.Pool, error) { + dsn, err := readSecret("DATABASE_URL", "DATABASE_URL_FILE") + if err != nil { + return nil, err + } + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + return nil, err + } + cfg.MaxConns = 10 + cfg.MinConns = 2 + cfg.MaxConnLifetime = 5 * time.Minute + cfg.ConnConfig.ConnectTimeout = 3 * time.Second + + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + return nil, err + } + pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := pool.Ping(pingCtx); err != nil { + pool.Close() + return nil, err + } + return pool, nil +} + +func dbCtx(parent context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(parent, 3*time.Second) +} diff --git a/evidence/attachments.txt b/evidence/attachments.txt new file mode 100644 index 00000000..8ec36d20 --- /dev/null +++ b/evidence/attachments.txt @@ -0,0 +1,3 @@ +# 2026-11-05T21:56:06Z $ python3 tests/attachments.py +PASS: attachment validation, limits and access control +# exit=0 diff --git a/evidence/failover-1.txt b/evidence/failover-1.txt new file mode 100644 index 00000000..7f7a0445 --- /dev/null +++ b/evidence/failover-1.txt @@ -0,0 +1,8 @@ +# 2026-11-05T21:57:18Z $ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 139, "failed_attempts": 1, "first_recovery_seconds": 3.475} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +# exit=0 +# api containers after hook +mtusi2026lab5-fork-api-1 Exited (0) 12 seconds ago +mtusi2026lab5-fork-api-2 Up 35 seconds (healthy) diff --git a/evidence/failover-2.txt b/evidence/failover-2.txt new file mode 100644 index 00000000..537bfe2c --- /dev/null +++ b/evidence/failover-2.txt @@ -0,0 +1,7 @@ +# 2026-11-05T22:01:19Z $ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 140, "failed_attempts": 0, "first_recovery_seconds": 3.465} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +# api containers after hook +mtusi2026lab5-fork-api-1 Up 35 seconds (healthy) +mtusi2026lab5-fork-api-2 Exited (0) 12 seconds ago diff --git a/evidence/failover-each.json b/evidence/failover-each.json new file mode 100644 index 00000000..1ecd3d31 --- /dev/null +++ b/evidence/failover-each.json @@ -0,0 +1,71 @@ +{ + "baseline": { + "window_seconds": 6, + "successful_requests": 3146, + "failed_attempts": 0, + "successful_rps": 524.3, + "last_error_offset_seconds": null, + "instances": [ + "4337e4656268", + "eb2165af370d" + ] + }, + "replicas": [ + { + "stopped_replica": "eb2165af370d", + "during_failure": { + "window_seconds": 15.35, + "successful_requests": 6738, + "failed_attempts": 2, + "successful_rps": 438.9, + "last_error_offset_seconds": 0.66, + "instances": [ + "4337e4656268", + "eb2165af370d" + ] + }, + "seen_stopped_replica_after_stop": false, + "stop_command_seconds": 3.35, + "restore_seconds": 6.48, + "after_restore": { + "window_seconds": 7.0, + "successful_requests": 3915, + "failed_attempts": 0, + "successful_rps": 559.3, + "last_error_offset_seconds": null, + "instances": [ + "4337e4656268", + "eb2165af370d" + ] + } + }, + { + "stopped_replica": "4337e4656268", + "during_failure": { + "window_seconds": 15.5, + "successful_requests": 6926, + "failed_attempts": 0, + "successful_rps": 446.9, + "last_error_offset_seconds": null, + "instances": [ + "4337e4656268", + "eb2165af370d" + ] + }, + "seen_stopped_replica_after_stop": false, + "stop_command_seconds": 3.5, + "restore_seconds": 6.52, + "after_restore": { + "window_seconds": 7.0, + "successful_requests": 3608, + "failed_attempts": 0, + "successful_rps": 515.4, + "last_error_offset_seconds": null, + "instances": [ + "4337e4656268", + "eb2165af370d" + ] + } + } + ] +} diff --git a/evidence/faults.txt b/evidence/faults.txt new file mode 100644 index 00000000..5a5824d3 --- /dev/null +++ b/evidence/faults.txt @@ -0,0 +1,9 @@ +# 2026-11-05T22:02:15Z $ python3 tests/faults.py +1) api crashed after COMMIT before 202/publish: attachment=50f6ace7-1393-4c19-b47a-1def389a62c3 recovered by outbox relay (published rows=1) -> ready attempt=1 +2) worker crashed after S3 write before COMMIT: lease expired, reaper requeued, attempt=2 ready, variants=3 +3) worker crashed after COMMIT before ack: message redelivered after restart, skipped as no-op, attempt=1, variants=3 +4) s3 down: transient errors retried with backoff+jitter delays_ms=[931, 1284, 2760, 7484], then failed retry_exhausted, listed in registry + replay: failed -> queued -> ready (audit rows=1); replay of ready job skipped, variants=3 +5) corrupted IDAT (passes header/tail check in API): failed invalid_image on attempt 1, no retries +PASS: outbox recovery, lease recovery, redelivery after commit, backoff with jitter, retry_exhausted, replay, permanent error +# exit=0 diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt new file mode 100644 index 00000000..0830e230 --- /dev/null +++ b/evidence/idempotency.txt @@ -0,0 +1,3 @@ +# 2026-11-05T21:56:12Z $ python3 tests/idempotency.py +PASS: 32 parallel requests, 1 message, 2 instances +# exit=0 diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..c70f73f7 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,25 @@ +# before +api Up 2 minutes (healthy) +api Up 2 minutes (healthy) +broker Up 16 minutes (healthy) +janitor Up 2 minutes +lb Up 12 minutes +postgres Up 19 minutes (healthy) +s3 Up 7 minutes (healthy) +valkey Up 19 minutes (healthy) +worker Up 37 seconds +# 2026-11-05T21:56:14Z $ make test-persistence ACTION_SCRIPT=scripts/restart.sh +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +# exit=0 +# after +api Up 2 seconds (health: starting) +api Up 2 seconds (health: starting) +broker Up 46 seconds (healthy) +janitor Up 5 seconds +lb Up 5 seconds +postgres Up 46 seconds (healthy) +s3 Up 36 seconds (healthy) +valkey Up 46 seconds (healthy) +worker Up Less than a second diff --git a/evidence/queue-recovery.txt b/evidence/queue-recovery.txt new file mode 100644 index 00000000..9b024075 --- /dev/null +++ b/evidence/queue-recovery.txt @@ -0,0 +1,7 @@ +# 2026-11-05T22:01:54Z $ python3 tests/queue_recovery.py +accepted: attachment=d4c108ad-3da8-467c-aee0-eb2ee25759fc job=0208b98f-c201-4686-aa66-62b0bbf703ae status=queued, job in stream, worker stopped +restarted broker and api: job still in stream, attachment still queued +worker started: attachment ready, attempt=1, variants=3 +duplicate delivery: skipped as no-op, acked, variants=3, updated_at unchanged +PASS: worker stop, broker/api restart, redelivery idempotency +# exit=0 diff --git a/evidence/rolling.json b/evidence/rolling.json new file mode 100644 index 00000000..ee0b09e9 --- /dev/null +++ b/evidence/rolling.json @@ -0,0 +1,24 @@ +{ + "old_replicas": [ + "eb2165af370d", + "4337e4656268" + ], + "new_replicas": [ + "2ce132adb5c9", + "2fe5ec734588" + ], + "update_window": { + "window_seconds": 16.4, + "successful_requests": 10091, + "failed_attempts": 4, + "successful_rps": 615.4, + "last_error_offset_seconds": 13.57, + "instances": [ + "2ce132adb5c9", + "2fe5ec734588", + "4337e4656268", + "eb2165af370d" + ] + }, + "old_replicas_served_after_update": [] +} diff --git a/evidence/s3-checks.txt b/evidence/s3-checks.txt new file mode 100644 index 00000000..4dcdd84c --- /dev/null +++ b/evidence/s3-checks.txt @@ -0,0 +1,3 @@ +# 2026-11-05T22:04:42Z $ python3 tests/s3_checks.py +PASS: private bucket, limited credentials, s3 outage, orphan and staging cleanup +# exit=0 diff --git a/evidence/session-revocation.txt b/evidence/session-revocation.txt new file mode 100644 index 00000000..38dab274 --- /dev/null +++ b/evidence/session-revocation.txt @@ -0,0 +1 @@ +PASS: token revoked while valkey was stopped stays revoked after valkey restart (401) diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..687a0371 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,6 @@ +# 2026-11-05T21:56:01Z $ sh -c make check && make test +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +# exit=0 diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 00000000..12f58996 --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,18 @@ +{ + "file_bytes": 9722608, + "concurrent_uploads": 12, + "rounds": 3, + "uploads_ok": 36, + "uploads_failed": 0, + "elapsed_seconds": 12.65, + "api_memory_limit_mib": 192, + "idle_memory_mib": { + "mtusi2026lab5-fork-api-1": 62.4, + "mtusi2026lab5-fork-api-2": 62.2 + }, + "peak_memory_mib": { + "mtusi2026lab5-fork-api-1": 118.8, + "mtusi2026lab5-fork-api-2": 99.5 + }, + "buffering_all_files_would_need_mib": 111.3 +} diff --git a/evidence/variants.txt b/evidence/variants.txt new file mode 100644 index 00000000..bf08edf0 --- /dev/null +++ b/evidence/variants.txt @@ -0,0 +1,3 @@ +# 2026-11-05T21:56:02Z $ python3 tests/variants.py +PASS: thumbnail/crop/optimized sizes, formats, checksums, no upscale, alpha on white, EXIF orientation, no EXIF in variants +# exit=0 diff --git a/evidence/worker-load.json b/evidence/worker-load.json new file mode 100644 index 00000000..00cc42e1 --- /dev/null +++ b/evidence/worker-load.json @@ -0,0 +1,26 @@ +{ + "huge_images": { + "count": 6, + "pixels": 39942400, + "bytes": 116545 + }, + "small_images": 20, + "worker_concurrency": 2, + "max_processing_jobs_observed": 2, + "all_ready": true, + "accept_seconds": 0.32, + "all_ready_seconds": 13.1, + "worker_peak_rss_mib": 616.7, + "worker_memory_limit_mib": 768, + "worker_restarts_oom": [ + "0", + "false" + ], + "timeout_case": { + "job_timeout_seconds": 1, + "status": "failed", + "error_code": "retry_exhausted", + "attempts": 5, + "last_error": "context deadline exceeded" + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..8f85fe5a --- /dev/null +++ b/go.mod @@ -0,0 +1,39 @@ +module mtusi-messenger + +go 1.26.0 + +require ( + github.com/gen2brain/webp v0.6.4 + github.com/jackc/pgx/v5 v5.11.0 + github.com/minio/minio-go/v7 v7.4.1 + github.com/redis/go-redis/v9 v9.23.0 + github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd + golang.org/x/crypto v0.58.0 + golang.org/x/image v0.47.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.10.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.uber.org/atomic v1.12.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/net v0.60.0 // indirect + golang.org/x/sync v0.24.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/text v0.43.0 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..c2500014 --- /dev/null +++ b/go.sum @@ -0,0 +1,85 @@ +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/gen2brain/webp v0.6.4 h1:SUDdmxADOAiPQ+5ylNmuHhuYf2dOi0KgKZHL5vpVCNU= +github.com/gen2brain/webp v0.6.4/go.mod h1:iGWMaCSw7t3I/Cv9llzEKmpnR36S8lS8VL/ZVjxU0JE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w= +github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= +github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= +go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo= +golang.org/x/image v0.47.0/go.mod h1:IkN5o4A7vq55vXN8FWjjnhg9Rs2GOtffqsSi9mWAKYE= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/handlers.go b/handlers.go new file mode 100644 index 00000000..0dadcec8 --- /dev/null +++ b/handlers.go @@ -0,0 +1,1051 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "regexp" + "strconv" + "strings" + "sync/atomic" + "time" + "unicode/utf8" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +var idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +var instanceID = func() string { + h, err := os.Hostname() + if err != nil || h == "" { + return newUUID() + } + return h +}() + +var usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +type apiError struct { + Code string `json:"code"` + Message string `json:"message"` + ReqID string `json:"request_id"` +} + +type errorBody struct { + Error apiError `json:"error"` +} + +type App struct { + db *pgxpool.Pool + log *slog.Logger + ready atomic.Bool + sessionTTL time.Duration + idempotencyTTL time.Duration + store *Store + limits limits + uploads chan struct{} + fault string +} + +func NewApp(db *pgxpool.Pool, log *slog.Logger, sessionTTL, idempotencyTTL time.Duration, store *Store, lim limits) *App { + a := &App{db: db, log: log, sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, store: store, limits: lim, uploads: make(chan struct{}, lim.uploadSlots), fault: os.Getenv("FAULT_POINT")} + a.ready.Store(true) + return a +} + +func (a *App) SetReady(v bool) { + a.ready.Store(v) +} + +func newUUID() string { + var b [16]byte + _, _ = rand.Read(b[:]) + b[6] = (b[6] & 0x0f) | 0x40 + b[8] = (b[8] & 0x3f) | 0x80 + return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]) +} + +func nowUTC() time.Time { + return time.Now().UTC().Truncate(time.Microsecond) +} + +func formatTime(t time.Time) string { + return t.UTC().Format("2006-01-02T15:04:05.000000Z") +} + +func parseTime(s string) (time.Time, error) { + return time.Parse("2006-01-02T15:04:05.000000Z", s) +} + +type statusWriter struct { + http.ResponseWriter + status int +} + +func (w *statusWriter) WriteHeader(code int) { + w.status = code + w.ResponseWriter.WriteHeader(code) +} + +type ctxKey int + +const reqIDKey ctxKey = 1 + +func (a *App) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + reqID := newUUID() + w.Header().Set("X-Request-Id", reqID) + w.Header().Set("X-Instance-Id", instanceID) + sw := &statusWriter{ResponseWriter: w, status: 200} + defer func() { + if rec := recover(); rec != nil { + a.writeErr(sw, reqID, 500, "internal_error", "internal error") + a.log.Error("panic", "request_id", reqID, "err", fmt.Sprint(rec)) + } + a.log.Info("request", + "method", r.Method, + "path", r.URL.Path, + "status", sw.status, + "duration_ms", time.Since(start).Milliseconds(), + "request_id", reqID, + ) + }() + next.ServeHTTP(sw, r.WithContext(context.WithValue(r.Context(), reqIDKey, reqID))) + }) +} + +func reqIDFrom(r *http.Request) string { + if v, ok := r.Context().Value(reqIDKey).(string); ok { + return v + } + return "" +} + +func (a *App) writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + _ = enc.Encode(v) +} + +func (a *App) writeErr(w http.ResponseWriter, reqID string, status int, code, msg string) { + a.writeJSON(w, status, errorBody{Error: apiError{Code: code, Message: msg, ReqID: reqID}}) +} + +func (a *App) writeUnauthorized(w http.ResponseWriter, r *http.Request, realm string) { + reqID := reqIDFrom(r) + if realm == "Basic" { + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + } else { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + } + a.writeErr(w, reqID, 401, "unauthorized", "unauthorized") +} + +func (a *App) Routes() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /health/live", a.handleLive) + mux.HandleFunc("GET /health/ready", a.handleReady) + mux.HandleFunc("POST /api/v1/users", a.handleCreateUser) + mux.HandleFunc("GET /api/v1/users/me", a.handleMe) + mux.HandleFunc("POST /api/v1/chats", a.handleCreateChat) + mux.HandleFunc("GET /api/v1/chats", a.handleListChats) + mux.HandleFunc("GET /api/v1/chats/{chat_id}", a.handleGetChat) + mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", a.handleCreateMessage) + mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", a.handleListMessages) + mux.HandleFunc("POST /api/v1/chats/{chat_id}/attachments", a.handleUploadAttachment) + mux.HandleFunc("GET /api/v1/attachments/{attachment_id}", a.handleGetAttachment) + mux.HandleFunc("GET /api/v1/attachments/{attachment_id}/content", a.handleDownloadAttachment) + mux.HandleFunc("POST /api/v1/auth/sessions", a.handleCreateSession) + mux.HandleFunc("DELETE /api/v1/auth/sessions/current", a.handleRevokeSession) + return a.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, pattern := mux.Handler(r) + if pattern == "" { + a.writeErr(w, reqIDFrom(r), 404, "not_found", "not found") + return + } + mux.ServeHTTP(w, r) + })) +} + +func (a *App) handleLive(w http.ResponseWriter, r *http.Request) { + a.writeJSON(w, 200, map[string]string{"status": "ok"}) +} + +func (a *App) handleReady(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + if !a.ready.Load() { + a.writeErr(w, reqID, 503, "unavailable", "shutting down") + return + } + ctx, cancel := dbCtx(r.Context()) + defer cancel() + if err := a.db.Ping(ctx); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 200, map[string]string{"status": "ok"}) +} + +func (a *App) readJSON(w http.ResponseWriter, r *http.Request, dst any) bool { + reqID := reqIDFrom(r) + ct := r.Header.Get("Content-Type") + media := strings.ToLower(strings.TrimSpace(strings.Split(ct, ";")[0])) + if media != "application/json" { + a.writeErr(w, reqID, 415, "unsupported_media_type", "Content-Type must be application/json") + return false + } + r.Body = http.MaxBytesReader(w, r.Body, 1<<20) + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + if _, ok := err.(*http.MaxBytesError); ok { + a.writeErr(w, reqID, 413, "payload_too_large", "body too large") + return false + } + a.writeErr(w, reqID, 400, "invalid_request", "invalid json") + return false + } + var extra json.RawMessage + if err := dec.Decode(&extra); err != io.EOF { + a.writeErr(w, reqID, 400, "invalid_request", "invalid json") + return false + } + return true +} + +func isBlank(s string) bool { + return strings.TrimSpace(s) == "" +} + +func isUUID(s string) bool { + if len(s) != 36 { + return false + } + for i, c := range s { + switch i { + case 8, 13, 18, 23: + if c != '-' { + return false + } + default: + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { + return false + } + } + } + raw := strings.ReplaceAll(s, "-", "") + _, err := hex.DecodeString(raw) + return err == nil +} + +func isPgCode(err error, code string) bool { + var pgErr *pgconn.PgError + return errors.As(err, &pgErr) && pgErr.Code == code +} + +func userJSON(u *User) map[string]any { + return map[string]any{ + "id": u.ID, + "username": u.Username, + "display_name": u.DisplayName, + "created_at": formatTime(u.CreatedAt), + } +} + +func (a *App) authUser(w http.ResponseWriter, r *http.Request) (*User, string) { + token, ok := parseBearer(r.Header.Get("Authorization")) + if !ok { + a.writeUnauthorized(w, r, "Bearer") + return nil, "" + } + hash, hashHex := tokenHash(token) + + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var u User + var expiresAt time.Time + err := a.db.QueryRow(ctx, ` + select u.id::text, u.username, u.display_name, u.password_hash, u.created_at, s.expires_at + from sessions s join users u on u.id = s.user_id + where s.token_hash = $1 and s.expires_at > now()`, hash).Scan( + &u.ID, &u.Username, &u.DisplayName, &u.PasswordHash, &u.CreatedAt, &expiresAt) + if errors.Is(err, pgx.ErrNoRows) { + a.writeUnauthorized(w, r, "Bearer") + return nil, "" + } + if err != nil { + a.writeErr(w, reqIDFrom(r), 503, "unavailable", "database unavailable") + return nil, "" + } + return &u, hashHex +} + +func (a *App) handleCreateUser(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + var body struct { + Username *string `json:"username"` + DisplayName *string `json:"display_name"` + Password *string `json:"password"` + } + if !a.readJSON(w, r, &body) { + return + } + if body.Username == nil || body.DisplayName == nil || body.Password == nil { + a.writeErr(w, reqID, 400, "invalid_request", "username, display_name and password required") + return + } + username := *body.Username + display := *body.DisplayName + password := *body.Password + if !usernameRe.MatchString(username) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid username") + return + } + n := utf8.RuneCountInString(display) + if n < 1 || n > 100 || isBlank(display) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid display_name") + return + } + if !validPassword(password) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid password") + return + } + ph, err := hashPassword(password) + if err != nil { + a.writeErr(w, reqID, 500, "internal_error", "internal error") + return + } + id := newUUID() + createdAt := nowUTC() + ctx, cancel := dbCtx(r.Context()) + defer cancel() + _, err = a.db.Exec(ctx, ` + insert into users(id, username, display_name, password_hash, created_at) + values($1::uuid, $2, $3, $4, $5)`, id, username, display, ph, createdAt) + if isPgCode(err, "23505") { + a.writeErr(w, reqID, 409, "conflict", "username taken") + return + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 201, map[string]any{ + "id": id, + "username": username, + "display_name": display, + "created_at": formatTime(createdAt), + }) +} + +func (a *App) handleCreateSession(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + username, password, ok := parseBasic(r.Header.Get("Authorization")) + if !ok { + a.writeUnauthorized(w, r, "Basic") + return + } + if !usernameRe.MatchString(username) || !validPassword(password) { + a.writeUnauthorized(w, r, "Basic") + return + } + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var u User + err := a.db.QueryRow(ctx, ` + select id::text, username, display_name, password_hash, created_at + from users where username = $1`, username).Scan( + &u.ID, &u.Username, &u.DisplayName, &u.PasswordHash, &u.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + a.writeUnauthorized(w, r, "Basic") + return + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + match, err := checkPassword(u.PasswordHash, password) + if err != nil || !match { + a.writeUnauthorized(w, r, "Basic") + return + } + token, hash, _, err := newToken() + if err != nil { + a.writeErr(w, reqID, 500, "internal_error", "internal error") + return + } + createdAt := nowUTC() + expiresAt := createdAt.Add(a.sessionTTL) + _, err = a.db.Exec(ctx, ` + insert into sessions(token_hash, user_id, created_at, expires_at) + values($1, $2::uuid, $3, $4)`, hash, u.ID, createdAt, expiresAt) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 201, map[string]any{ + "token": token, + "token_type": "Bearer", + "expires_at": formatTime(expiresAt), + "user": userJSON(&u), + }) +} + +func (a *App) handleRevokeSession(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + token, ok := parseBearer(r.Header.Get("Authorization")) + if !ok { + a.writeUnauthorized(w, r, "Bearer") + return + } + hash, _ := tokenHash(token) + ctx, cancel := dbCtx(r.Context()) + defer cancel() + tag, err := a.db.Exec(ctx, ` + delete from sessions where token_hash = $1 and expires_at > now()`, hash) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if tag.RowsAffected() == 0 { + a.writeUnauthorized(w, r, "Bearer") + return + } + w.Header().Set("X-Request-Id", reqID) + w.WriteHeader(204) +} + +func (a *App) handleMe(w http.ResponseWriter, r *http.Request) { + u, _ := a.authUser(w, r) + if u == nil { + return + } + a.writeJSON(w, 200, userJSON(u)) +} + +func (a *App) handleCreateChat(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + var body struct { + Title *string `json:"title"` + MemberIDs *[]string `json:"member_ids"` + } + if !a.readJSON(w, r, &body) { + return + } + if body.Title == nil || body.MemberIDs == nil { + a.writeErr(w, reqID, 400, "invalid_request", "title and member_ids required") + return + } + title := *body.Title + n := utf8.RuneCountInString(title) + if n < 1 || n > 100 || isBlank(title) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid title") + return + } + members := *body.MemberIDs + if len(members) < 1 || len(members) > 100 { + a.writeErr(w, reqID, 400, "invalid_request", "invalid member_ids") + return + } + seen := make(map[string]struct{}, len(members)) + hasCaller := false + for _, id := range members { + if !isUUID(id) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid member id") + return + } + if _, dup := seen[id]; dup { + a.writeErr(w, reqID, 400, "invalid_request", "duplicate member") + return + } + seen[id] = struct{}{} + if id == u.ID { + hasCaller = true + } + } + if !hasCaller { + a.writeErr(w, reqID, 400, "invalid_request", "caller must be member") + return + } + + chatID := newUUID() + createdAt := nowUTC() + ctx, cancel := dbCtx(r.Context()) + defer cancel() + tx, err := a.db.Begin(ctx) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + defer tx.Rollback(ctx) + + _, err = tx.Exec(ctx, `insert into chats(id, title, created_at) values($1::uuid, $2, $3)`, + chatID, title, createdAt) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + for _, mid := range members { + _, err = tx.Exec(ctx, `insert into chat_members(chat_id, user_id) values($1::uuid, $2::uuid)`, + chatID, mid) + if isPgCode(err, "23503") { + a.writeErr(w, reqID, 400, "invalid_request", "unknown member") + return + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + } + if err := tx.Commit(ctx); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 201, map[string]any{ + "id": chatID, + "title": title, + "member_ids": members, + "created_at": formatTime(createdAt), + }) +} + +func encodeCursor(parts ...string) string { + return base64.RawURLEncoding.EncodeToString([]byte(strings.Join(parts, "|"))) +} + +func decodeCursor(s string) ([]string, error) { + b, err := base64.RawURLEncoding.DecodeString(s) + if err != nil { + return nil, err + } + return strings.Split(string(b), "|"), nil +} + +func parseLimit(r *http.Request) (int, error) { + q := r.URL.Query().Get("limit") + if q == "" { + return 50, nil + } + n, err := strconv.Atoi(q) + if err != nil || n < 1 || n > 100 { + return 0, fmt.Errorf("bad limit") + } + return n, nil +} + +func (a *App) loadChatMembers(ctx context.Context, chatID string) ([]string, error) { + ctx, cancel := dbCtx(ctx) + defer cancel() + rows, err := a.db.Query(ctx, ` + select user_id::text from chat_members where chat_id = $1::uuid order by user_id`, chatID) + if err != nil { + return nil, err + } + defer rows.Close() + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + return ids, rows.Err() +} + +func (a *App) isMember(ctx context.Context, chatID, userID string) (bool, error) { + ctx, cancel := dbCtx(ctx) + defer cancel() + var ok bool + err := a.db.QueryRow(ctx, ` + select exists( + select 1 from chat_members where chat_id = $1::uuid and user_id = $2::uuid + )`, chatID, userID).Scan(&ok) + return ok, err +} + +func (a *App) handleListChats(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + limit, err := parseLimit(r) + if err != nil { + a.writeErr(w, reqID, 400, "invalid_request", "invalid limit") + return + } + var afterTS *time.Time + var afterID string + if cur := r.URL.Query().Get("cursor"); cur != "" { + parts, err := decodeCursor(cur) + if err != nil || len(parts) != 3 { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + if parts[0] != u.ID { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + ts, err := parseTime(parts[1]) + if err != nil || !isUUID(parts[2]) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + afterTS = &ts + afterID = parts[2] + } + + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var rows pgx.Rows + if afterTS == nil { + rows, err = a.db.Query(ctx, ` + select c.id::text, c.title, c.created_at + from chats c + join chat_members m on m.chat_id = c.id + where m.user_id = $1::uuid + order by c.created_at, c.id + limit $2`, u.ID, limit+1) + } else { + rows, err = a.db.Query(ctx, ` + select c.id::text, c.title, c.created_at + from chats c + join chat_members m on m.chat_id = c.id + where m.user_id = $1::uuid + and (c.created_at, c.id) > ($2, $3::uuid) + order by c.created_at, c.id + limit $4`, u.ID, *afterTS, afterID, limit+1) + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + defer rows.Close() + + type row struct { + id, title string + created time.Time + } + var list []row + for rows.Next() { + var x row + if err := rows.Scan(&x.id, &x.title, &x.created); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + list = append(list, x) + } + if err := rows.Err(); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + hasMore := len(list) > limit + if hasMore { + list = list[:limit] + } + out := make([]map[string]any, 0, len(list)) + for _, c := range list { + members, err := a.loadChatMembers(r.Context(), c.id) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + out = append(out, map[string]any{ + "id": c.id, + "title": c.title, + "member_ids": members, + "created_at": formatTime(c.created), + }) + } + var next any = nil + if hasMore && len(list) > 0 { + last := list[len(list)-1] + next = encodeCursor(u.ID, formatTime(last.created), last.id) + } + a.writeJSON(w, 200, map[string]any{"items": out, "next_cursor": next}) +} + +func (a *App) handleGetChat(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + chatID := r.PathValue("chat_id") + if !isUUID(chatID) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id") + return + } + ok, err := a.isMember(r.Context(), chatID, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if !ok { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var title string + var createdAt time.Time + err = a.db.QueryRow(ctx, `select title, created_at from chats where id = $1::uuid`, chatID). + Scan(&title, &createdAt) + if errors.Is(err, pgx.ErrNoRows) { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + members, err := a.loadChatMembers(r.Context(), chatID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 200, map[string]any{ + "id": chatID, + "title": title, + "member_ids": members, + "created_at": formatTime(createdAt), + }) +} + +func (a *App) handleCreateMessage(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + chatID := r.PathValue("chat_id") + if !isUUID(chatID) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id") + return + } + key := r.Header.Get("Idempotency-Key") + if !idempotencyKeyRe.MatchString(key) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid Idempotency-Key") + return + } + ok, err := a.isMember(r.Context(), chatID, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if !ok { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + var body struct { + Text *string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + } + if !a.readJSON(w, r, &body) { + return + } + if body.Text == nil { + a.writeErr(w, reqID, 400, "invalid_request", "text required") + return + } + text := *body.Text + attachmentIDs := body.AttachmentIDs + if attachmentIDs == nil { + attachmentIDs = []string{} + } + if len(attachmentIDs) > 10 { + a.writeErr(w, reqID, 400, "invalid_request", "too many attachments") + return + } + seen := make(map[string]bool, len(attachmentIDs)) + for _, id := range attachmentIDs { + if !isUUID(id) || seen[id] { + a.writeErr(w, reqID, 400, "invalid_request", "invalid attachment_ids") + return + } + seen[id] = true + } + n := utf8.RuneCountInString(text) + if n > 4000 || (len(attachmentIDs) == 0 && (n < 1 || isBlank(text))) || (n > 0 && isBlank(text)) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid text") + return + } + if len(attachmentIDs) > 0 { + var found, failed int + cctx, ccancel := dbCtx(r.Context()) + err := a.db.QueryRow(cctx, ` + select count(*), count(*) filter (where status = 'failed') + from attachments where id = any($1::uuid[]) and chat_id = $2::uuid`, attachmentIDs, chatID).Scan(&found, &failed) + ccancel() + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if found != len(attachmentIDs) { + a.writeErr(w, reqID, 404, "not_found", "attachment not found") + return + } + if failed > 0 { + a.writeErr(w, reqID, 409, "conflict", "attachment processing failed") + return + } + } + hashInput := text + if len(attachmentIDs) > 0 { + hashInput += "\x00" + strings.Join(attachmentIDs, ",") + } + sum := sha256.Sum256([]byte(hashInput)) + bodyHash := hex.EncodeToString(sum[:]) + path := r.URL.Path + ctx, cancel := dbCtx(r.Context()) + defer cancel() + tx, err := a.db.Begin(ctx) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + defer tx.Rollback(ctx) + + if _, err := tx.Exec(ctx, ` + delete from idempotency_keys + where user_id=$1::uuid and method='POST' and path=$2 and key=$3 + and created_at < $4`, + u.ID, path, key, time.Now().Add(-a.idempotencyTTL)); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + msgID := newUUID() + createdAt := nowUTC() + resp := map[string]any{ + "id": msgID, + "chat_id": chatID, + "sender_id": u.ID, + "text": text, + "created_at": formatTime(createdAt), + "attachment_ids": attachmentIDs, + } + respJSON, _ := json.Marshal(resp) + tag, err := tx.Exec(ctx, ` + insert into idempotency_keys(user_id, method, path, key, body_hash, response, created_at) + values($1::uuid, 'POST', $2, $3, $4, $5, $6) + on conflict do nothing`, + u.ID, path, key, bodyHash, respJSON, createdAt) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if tag.RowsAffected() == 0 { + var storedHash string + var stored []byte + err := tx.QueryRow(ctx, ` + select body_hash, response from idempotency_keys + where user_id=$1::uuid and method='POST' and path=$2 and key=$3`, + u.ID, path, key).Scan(&storedHash, &stored) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if storedHash != bodyHash { + a.writeErr(w, reqID, 409, "conflict", "idempotency key reused with different body") + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(201) + _, _ = w.Write(append(stored, '\n')) + return + } + if _, err := tx.Exec(ctx, ` + insert into messages(id, chat_id, sender_id, text, created_at) + values($1::uuid, $2::uuid, $3::uuid, $4, $5)`, + msgID, chatID, u.ID, text, createdAt); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if len(attachmentIDs) > 0 { + if _, err := tx.Exec(ctx, ` + insert into message_attachments(message_id, attachment_id, position) + select $1::uuid, t.id, t.ord - 1 + from unnest($2::uuid[]) with ordinality as t(id, ord)`, + msgID, attachmentIDs); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + } + if err := tx.Commit(ctx); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 201, resp) +} + +func (a *App) handleListMessages(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + chatID := r.PathValue("chat_id") + if !isUUID(chatID) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id") + return + } + ok, err := a.isMember(r.Context(), chatID, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if !ok { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + limit, err := parseLimit(r) + if err != nil { + a.writeErr(w, reqID, 400, "invalid_request", "invalid limit") + return + } + var afterTS *time.Time + var afterID string + if cur := r.URL.Query().Get("cursor"); cur != "" { + parts, err := decodeCursor(cur) + if err != nil || len(parts) != 3 { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + if parts[0] != chatID { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + ts, err := parseTime(parts[1]) + if err != nil || !isUUID(parts[2]) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + afterTS = &ts + afterID = parts[2] + } + + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var rows pgx.Rows + if afterTS == nil { + rows, err = a.db.Query(ctx, ` + select id::text, chat_id::text, sender_id::text, text, created_at + from messages + where chat_id = $1::uuid + order by created_at, id + limit $2`, chatID, limit+1) + } else { + rows, err = a.db.Query(ctx, ` + select id::text, chat_id::text, sender_id::text, text, created_at + from messages + where chat_id = $1::uuid + and (created_at, id) > ($2, $3::uuid) + order by created_at, id + limit $4`, chatID, *afterTS, afterID, limit+1) + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + defer rows.Close() + + type row struct { + id, chatID, senderID, text string + created time.Time + } + var list []row + for rows.Next() { + var x row + if err := rows.Scan(&x.id, &x.chatID, &x.senderID, &x.text, &x.created); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + list = append(list, x) + } + if err := rows.Err(); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + hasMore := len(list) > limit + if hasMore { + list = list[:limit] + } + attachments := map[string][]string{} + if len(list) > 0 { + ids := make([]string, len(list)) + for i, m := range list { + ids[i] = m.id + } + arows, err := a.db.Query(ctx, ` + select message_id::text, attachment_id::text + from message_attachments + where message_id = any($1::uuid[]) + order by message_id, position`, ids) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + for arows.Next() { + var mid, aid string + if err := arows.Scan(&mid, &aid); err != nil { + arows.Close() + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + attachments[mid] = append(attachments[mid], aid) + } + arows.Close() + if err := arows.Err(); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + } + out := make([]map[string]any, 0, len(list)) + for _, m := range list { + ids := attachments[m.id] + if ids == nil { + ids = []string{} + } + out = append(out, map[string]any{ + "id": m.id, + "chat_id": m.chatID, + "sender_id": m.senderID, + "text": m.text, + "created_at": formatTime(m.created), + "attachment_ids": ids, + }) + } + var next any = nil + if hasMore && len(list) > 0 { + last := list[len(list)-1] + next = encodeCursor(chatID, formatTime(last.created), last.id) + } + a.writeJSON(w, 200, map[string]any{"items": out, "next_cursor": next}) +} + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} diff --git a/imaging.go b/imaging.go new file mode 100644 index 00000000..6ccc5f28 --- /dev/null +++ b/imaging.go @@ -0,0 +1,176 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + "image/draw" + "image/jpeg" + _ "image/png" + + "github.com/gen2brain/webp" + "github.com/rwcarlsen/goexif/exif" + xdraw "golang.org/x/image/draw" +) + +const ( + jpegQuality = 85 + webpQuality = 80 + webpMethod = 4 +) + +var errInvalidImage = errors.New("invalid image") + +type variantSpec struct { + name, contentType, ext string + box int + crop bool +} + +var variantSpecs = []variantSpec{ + {"thumbnail", "image/jpeg", "jpg", 256, false}, + {"crop", "image/jpeg", "jpg", 128, true}, + {"optimized", "image/webp", "webp", 1280, false}, +} + +type rendered struct { + spec variantSpec + data []byte + width, height int + sha256 string +} + +func renderVariants(ctx context.Context, data []byte, maxPixels int64) ([]rendered, error) { + cfg, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || cfg.Width < 1 || cfg.Height < 1 { + return nil, errInvalidImage + } + if int64(cfg.Width)*int64(cfg.Height) > maxPixels { + return nil, errInvalidImage + } + src, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return nil, errInvalidImage + } + orient := 1 + if format == "jpeg" { + orient = orientation(data) + } + src = shrink(src, 1280) + var out []rendered + for _, spec := range variantSpecs { + if err := ctx.Err(); err != nil { + return nil, err + } + img := render(src, cfg.Width, cfg.Height, orient, spec) + var buf bytes.Buffer + if spec.contentType == "image/webp" { + err = webp.Encode(&buf, img, webp.Options{Quality: webpQuality, Method: webpMethod}) + } else { + err = jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality}) + } + if err != nil { + return nil, err + } + sum := sha256.Sum256(buf.Bytes()) + b := img.Bounds() + out = append(out, rendered{spec: spec, data: buf.Bytes(), width: b.Dx(), height: b.Dy(), sha256: hex.EncodeToString(sum[:])}) + } + return out, nil +} + +func orientation(data []byte) int { + x, err := exif.Decode(bytes.NewReader(data)) + if err != nil { + return 1 + } + tag, err := x.Get(exif.Orientation) + if err != nil { + return 1 + } + v, err := tag.Int(0) + if err != nil || v < 1 || v > 8 { + return 1 + } + return v +} + +func fit(w, h, box int) (int, int) { + if w <= box && h <= box { + return w, h + } + if w >= h { + return box, max(1, h*box/w) + } + return max(1, w*box/h), box +} + +func shrink(src image.Image, limit int) image.Image { + for { + b := src.Bounds() + if max(b.Dx(), b.Dy())/2 < limit { + return src + } + dst := image.NewRGBA(image.Rect(0, 0, max(1, b.Dx()/2), max(1, b.Dy()/2))) + xdraw.ApproxBiLinear.Scale(dst, dst.Bounds(), src, b, xdraw.Src, nil) + src = dst + } +} + +func render(src image.Image, sw, sh, orient int, spec variantSpec) *image.RGBA { + b := src.Bounds() + rect := b + var tw, th int + if spec.crop { + side := min(b.Dx(), b.Dy()) + x0, y0 := b.Min.X+(b.Dx()-side)/2, b.Min.Y+(b.Dy()-side)/2 + rect = image.Rect(x0, y0, x0+side, y0+side) + tw, th = spec.box, spec.box + } else if orient >= 5 { + th, tw = fit(sh, sw, spec.box) + } else { + tw, th = fit(sw, sh, spec.box) + } + dst := image.NewRGBA(image.Rect(0, 0, tw, th)) + draw.Draw(dst, dst.Bounds(), image.White, image.Point{}, draw.Src) + xdraw.CatmullRom.Scale(dst, dst.Bounds(), src, rect, xdraw.Over, nil) + return orientImage(dst, orient) +} + +func orientImage(img *image.RGBA, o int) *image.RGBA { + if o <= 1 { + return img + } + w, h := img.Bounds().Dx(), img.Bounds().Dy() + dw, dh := w, h + if o >= 5 { + dw, dh = h, w + } + dst := image.NewRGBA(image.Rect(0, 0, dw, dh)) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + var nx, ny int + switch o { + case 2: + nx, ny = w-1-x, y + case 3: + nx, ny = w-1-x, h-1-y + case 4: + nx, ny = x, h-1-y + case 5: + nx, ny = y, x + case 6: + nx, ny = h-1-y, x + case 7: + nx, ny = h-1-y, w-1-x + case 8: + nx, ny = y, w-1-x + } + copy(dst.Pix[dst.PixOffset(nx, ny):dst.PixOffset(nx, ny)+4], img.Pix[img.PixOffset(x, y):img.PixOffset(x, y)+4]) + } + } + return dst +} diff --git a/lb/haproxy.cfg b/lb/haproxy.cfg new file mode 100644 index 00000000..d63fed5b --- /dev/null +++ b/lb/haproxy.cfg @@ -0,0 +1,38 @@ +global + log stdout format raw local0 + maxconn 2000 + +resolvers docker + nameserver dns 127.0.0.11:53 + accepted_payload_size 8192 + hold valid 1s + hold nx 1s + hold obsolete 1s + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + +defaults + mode http + log global + option httplog + option redispatch 1 + retries 3 + retry-on conn-failure empty-response + timeout connect 1s + timeout client 20s + timeout server 15s + timeout http-request 10s + timeout queue 5s + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr none on-marked-down shutdown-sessions + server-template api 1-8 api:8080 diff --git a/main.go b/main.go new file mode 100644 index 00000000..a3383716 --- /dev/null +++ b/main.go @@ -0,0 +1,196 @@ +package main + +import ( + "context" + "encoding/json" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func main() { + log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})) + + host := envOr("APP_HOST", "0.0.0.0") + port := envOr("APP_PORT", "8080") + shutdownSec, _ := strconv.Atoi(envOr("SHUTDOWN_TIMEOUT_SECONDS", "5")) + if shutdownSec < 1 { + shutdownSec = 5 + } + ttlSec, _ := strconv.Atoi(envOr("SESSION_TTL_SECONDS", "3600")) + if ttlSec < 1 { + ttlSec = 3600 + } + + idemSec, _ := strconv.Atoi(envOr("IDEMPOTENCY_TTL_SECONDS", "86400")) + if idemSec < 1 { + idemSec = 86400 + } + drainSec, _ := strconv.Atoi(envOr("DRAIN_SECONDS", "3")) + + ctx := context.Background() + if len(os.Args) > 1 && os.Args[1] == "s3-init" { + if err := s3Init(ctx); err != nil { + log.Error("s3 init failed", "err", err.Error()) + os.Exit(1) + } + log.Info("bucket ready") + return + } + db, err := openDB(ctx) + if err != nil { + log.Error("database open failed", "err", err.Error()) + os.Exit(1) + } + defer db.Close() + + if len(os.Args) > 1 && os.Args[1] == "migrate" { + if err := migrate(ctx, db); err != nil { + log.Error("migrate failed", "err", err.Error()) + os.Exit(1) + } + log.Info("migrations applied") + return + } + + store, err := openStore() + if err != nil { + log.Error("s3 open failed", "err", err.Error()) + os.Exit(1) + } + + if len(os.Args) > 1 && (os.Args[1] == "cleanup" || os.Args[1] == "janitor") { + runCleanup(ctx, db, store, log, os.Args[1] == "janitor") + return + } + + maxBytes, _ := strconv.ParseInt(envOr("MAX_UPLOAD_BYTES", "10485760"), 10, 64) + maxPixels, _ := strconv.ParseInt(envOr("MAX_IMAGE_PIXELS", "40000000"), 10, 64) + maxAttempts := envInt("JOB_MAX_ATTEMPTS", 5) + + if len(os.Args) > 1 && (os.Args[1] == "replay" || os.Args[1] == "failed") { + out := json.NewEncoder(os.Stdout) + if os.Args[1] == "failed" { + items, err := listFailed(ctx, db) + if err != nil { + log.Error("failed jobs query failed", "err", err.Error()) + os.Exit(1) + } + for _, item := range items { + _ = out.Encode(item) + } + return + } + if len(os.Args) != 3 || !isUUID(os.Args[2]) { + log.Error("usage: replay ") + os.Exit(2) + } + result, err := replayJob(ctx, db, os.Args[2], maxAttempts) + if err != nil { + log.Error("replay failed", "err", err.Error()) + os.Exit(1) + } + _ = out.Encode(map[string]string{"attachment_id": os.Args[2], "result": result}) + return + } + + broker, err := openBroker() + if err != nil { + log.Error("broker open failed", "err", err.Error()) + os.Exit(1) + } + + if len(os.Args) > 1 && os.Args[1] == "worker" { + timeout := time.Duration(envInt("JOB_TIMEOUT_SECONDS", 60)) * time.Second + owner, _ := os.Hostname() + w := &Worker{ + db: db, store: store, broker: broker, log: log, + owner: owner + "-" + newUUID()[:8], + timeout: timeout, + lease: timeout + 15*time.Second, + backoffBase: time.Duration(envInt("JOB_BACKOFF_BASE_MS", 1000)) * time.Millisecond, + backoffMax: time.Duration(envInt("JOB_BACKOFF_MAX_MS", 30000)) * time.Millisecond, + maxBytes: maxBytes, + maxPixels: maxPixels, + fault: os.Getenv("FAULT_POINT"), + } + runWorker(db, store, broker, log, w, envInt("WORKER_CONCURRENCY", 2)) + return + } + + slots := envInt("UPLOAD_CONCURRENCY", 10) + app := NewApp(db, log, time.Duration(ttlSec)*time.Second, time.Duration(idemSec)*time.Second, store, + limits{maxBytes: maxBytes, maxPixels: maxPixels, uploadSlots: slots, maxAttempts: maxAttempts}) + relayCtx, stopRelay := context.WithCancel(ctx) + defer stopRelay() + go runRelay(relayCtx, db, broker, log) + + addr := net.JoinHostPort(host, port) + srv := &http.Server{ + Addr: addr, + Handler: app.Routes(), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 15 * time.Second, + IdleTimeout: 30 * time.Second, + } + + go func() { + log.Info("listening", "addr", addr, "env", envOr("APP_ENV", "lab"), "session_ttl_s", ttlSec) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Error("server error", "err", err.Error()) + os.Exit(1) + } + }() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) + sig := <-stop + log.Info("shutdown started", "signal", sig.String()) + app.SetReady(false) + time.Sleep(time.Duration(drainSec) * time.Second) + + shCtx, cancel := context.WithTimeout(context.Background(), time.Duration(shutdownSec)*time.Second) + defer cancel() + if err := srv.Shutdown(shCtx); err != nil { + log.Error("shutdown error", "err", err.Error()) + os.Exit(1) + } + log.Info("shutdown finished") +} + +func runCleanup(ctx context.Context, db *pgxpool.Pool, store *Store, log *slog.Logger, loop bool) { + graceSec, _ := strconv.Atoi(envOr("CLEANUP_GRACE_SECONDS", "600")) + intervalSec, _ := strconv.Atoi(envOr("CLEANUP_INTERVAL_SECONDS", "300")) + retention := time.Duration(envInt("STAGING_RETENTION_SECONDS", 86400)) * time.Second + for { + removed, err := cleanupOrphans(ctx, db, store, time.Duration(graceSec)*time.Second, retention, log) + if err != nil { + log.Error("cleanup failed", "err", err.Error()) + if !loop { + os.Exit(1) + } + } else { + log.Info("cleanup finished", "removed", removed) + } + if !loop { + return + } + time.Sleep(time.Duration(intervalSec) * time.Second) + } +} + +func envInt(key string, def int) int { + v, err := strconv.Atoi(os.Getenv(key)) + if err != nil || v < 1 { + return def + } + return v +} diff --git a/migrate.go b/migrate.go new file mode 100644 index 00000000..d2b953fe --- /dev/null +++ b/migrate.go @@ -0,0 +1,84 @@ +package main + +import ( + "context" + "embed" + "fmt" + "sort" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +//go:embed migrations/*.sql +var migrationFS embed.FS + +func migrate(ctx context.Context, pool *pgxpool.Pool) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + conn, err := pool.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + if _, err := conn.Exec(ctx, `select pg_advisory_lock(727001)`); err != nil { + return err + } + defer conn.Exec(context.Background(), `select pg_advisory_unlock(727001)`) + + _, err = conn.Exec(ctx, ` + create table if not exists schema_migrations ( + version text primary key, + applied_at timestamptz not null + )`) + if err != nil { + return fmt.Errorf("schema_migrations: %w", err) + } + + entries, err := migrationFS.ReadDir("migrations") + if err != nil { + return err + } + var names []string + for _, e := range entries { + if !e.IsDir() && strings.HasSuffix(e.Name(), ".sql") { + names = append(names, e.Name()) + } + } + sort.Strings(names) + + for _, name := range names { + var exists bool + err := conn.QueryRow(ctx, `select exists(select 1 from schema_migrations where version=$1)`, name).Scan(&exists) + if err != nil { + return err + } + if exists { + continue + } + + body, err := migrationFS.ReadFile("migrations/" + name) + if err != nil { + return err + } + + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + if _, err := tx.Exec(ctx, string(body)); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("migration %s: %w", name, err) + } + if _, err := tx.Exec(ctx, `insert into schema_migrations(version, applied_at) values($1, now())`, name); err != nil { + _ = tx.Rollback(ctx) + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + } + return nil +} diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 00000000..a50290e5 --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,38 @@ +create table users ( + id uuid primary key, + username text not null unique, + display_name text not null, + password_hash text not null, + created_at timestamptz not null +); + +create table chats ( + id uuid primary key, + title text not null, + created_at timestamptz not null +); + +create table chat_members ( + chat_id uuid not null references chats(id), + user_id uuid not null references users(id), + primary key (chat_id, user_id) +); + +create index chat_members_user_idx on chat_members (user_id); + +create table messages ( + id uuid primary key, + chat_id uuid not null references chats(id), + sender_id uuid not null references users(id), + text text not null, + created_at timestamptz not null +); + +create index messages_history_idx on messages (chat_id, created_at, id); + +create table sessions ( + token_hash bytea primary key, + user_id uuid not null references users(id) on delete cascade, + created_at timestamptz not null, + expires_at timestamptz not null +); diff --git a/migrations/0002_idempotency.sql b/migrations/0002_idempotency.sql new file mode 100644 index 00000000..0cfbc2d6 --- /dev/null +++ b/migrations/0002_idempotency.sql @@ -0,0 +1,10 @@ +create table idempotency_keys ( + user_id uuid not null references users(id) on delete cascade, + method text not null, + path text not null, + key text not null, + body_hash text not null, + response jsonb not null, + created_at timestamptz not null, + primary key (user_id, method, path, key) +); diff --git a/migrations/0003_attachments.sql b/migrations/0003_attachments.sql new file mode 100644 index 00000000..00d00fbe --- /dev/null +++ b/migrations/0003_attachments.sql @@ -0,0 +1,22 @@ +create table attachments ( + id uuid primary key, + chat_id uuid not null references chats(id), + owner_id uuid not null references users(id), + filename text not null, + content_type text not null, + size_bytes bigint not null, + sha256 text not null, + object_key text not null unique, + created_at timestamptz not null +); + +create index attachments_chat_idx on attachments (chat_id); + +create table message_attachments ( + message_id uuid not null references messages(id), + attachment_id uuid not null references attachments(id), + position int not null, + primary key (message_id, attachment_id) +); + +create index message_attachments_attachment_idx on message_attachments (attachment_id); diff --git a/migrations/0004_jobs.sql b/migrations/0004_jobs.sql new file mode 100644 index 00000000..feddbf0a --- /dev/null +++ b/migrations/0004_jobs.sql @@ -0,0 +1,60 @@ +alter table attachments + add column status text not null default 'ready', + add column error_code text, + add column staging_key text, + add column updated_at timestamptz; + +update attachments set updated_at = created_at; + +alter table attachments + alter column status drop default, + alter column updated_at set not null, + alter column object_key drop not null; + +create table attachment_variants ( + attachment_id uuid not null references attachments(id), + pipeline_version int not null, + name text not null, + object_key text not null unique, + content_type text not null, + size_bytes bigint not null, + width int not null, + height int not null, + sha256 text not null, + primary key (attachment_id, pipeline_version, name) +); + +create table jobs ( + id uuid primary key, + attachment_id uuid not null unique references attachments(id), + pipeline_version int not null, + status text not null, + attempt int not null, + max_attempts int not null, + next_attempt_at timestamptz not null, + lease_owner text, + lease_until timestamptz, + last_error text, + created_at timestamptz not null, + updated_at timestamptz not null +); + +create index jobs_lease_idx on jobs (lease_until) where status = 'processing'; + +create table outbox ( + id bigserial primary key, + job_id uuid not null references jobs(id), + attempt int not null, + available_at timestamptz not null, + published_at timestamptz, + created_at timestamptz not null +); + +create index outbox_pending_idx on outbox (available_at) where published_at is null; + +create table job_replays ( + id bigserial primary key, + job_id uuid not null references jobs(id), + previous_error text, + replayed_at timestamptz not null +); diff --git a/models.go b/models.go new file mode 100644 index 00000000..c6fd8042 --- /dev/null +++ b/models.go @@ -0,0 +1,33 @@ +package main + +import "time" + +type User struct { + ID string + Username string + DisplayName string + PasswordHash string + CreatedAt time.Time +} + +type Chat struct { + ID string + Title string + MemberIDs []string + CreatedAt time.Time +} + +type Message struct { + ID string + ChatID string + SenderID string + Text string + CreatedAt time.Time +} + +type Session struct { + TokenHash []byte + UserID string + CreatedAt time.Time + ExpiresAt time.Time +} diff --git a/s3.go b/s3.go new file mode 100644 index 00000000..5f4e9bdd --- /dev/null +++ b/s3.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "io" + "net" + "net/http" + "net/url" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type Store struct { + client *minio.Client + bucket string +} + +func openStore() (*Store, error) { + endpoint, err := url.Parse(envOr("S3_ENDPOINT", "http://s3:9000")) + if err != nil { + return nil, err + } + keyID, err := readSecret("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE") + if err != nil { + return nil, err + } + secret, err := readSecret("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE") + if err != nil { + return nil, err + } + transport := &http.Transport{ + DialContext: (&net.Dialer{Timeout: 2 * time.Second}).DialContext, + ResponseHeaderTimeout: 10 * time.Second, + MaxIdleConnsPerHost: 16, + IdleConnTimeout: 30 * time.Second, + } + client, err := minio.New(endpoint.Host, &minio.Options{ + Creds: credentials.NewStaticV4(keyID, secret, ""), + Secure: endpoint.Scheme == "https", + Region: envOr("S3_REGION", "us-east-1"), + Transport: transport, + MaxRetries: 1, + }) + if err != nil { + return nil, err + } + return &Store{client: client, bucket: envOr("S3_BUCKET", "messenger-attachments")}, nil +} + +func (s *Store) Put(ctx context.Context, key string, r io.Reader, size int64, contentType string) error { + _, err := s.client.PutObject(ctx, s.bucket, key, r, size, minio.PutObjectOptions{ + ContentType: contentType, + PartSize: 5 << 20, + }) + return err +} + +func (s *Store) Get(ctx context.Context, key string) (*minio.Object, minio.ObjectInfo, error) { + obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, minio.ObjectInfo{}, err + } + info, err := obj.Stat() + if err != nil { + obj.Close() + return nil, minio.ObjectInfo{}, err + } + return obj, info, nil +} + +func (s *Store) Remove(ctx context.Context, key string) error { + return s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}) +} + +func (s *Store) List(ctx context.Context, prefix string) <-chan minio.ObjectInfo { + return s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) +} + +func isNoSuchKey(err error) bool { + return minio.ToErrorResponse(err).Code == "NoSuchKey" +} + +func s3Init(ctx context.Context) error { + s, err := openStore() + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + for { + exists, err := s.client.BucketExists(ctx, s.bucket) + if err == nil { + if exists { + return nil + } + err = s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: envOr("S3_REGION", "us-east-1")}) + if err == nil || minio.ToErrorResponse(err).Code == "BucketAlreadyOwnedByYou" { + return nil + } + } + select { + case <-ctx.Done(): + return err + case <-time.After(2 * time.Second): + } + } +} diff --git a/scripts/mksecrets.sh b/scripts/mksecrets.sh new file mode 100755 index 00000000..a036162f --- /dev/null +++ b/scripts/mksecrets.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +mkdir -p "$ROOT/secrets" +cd "$ROOT/secrets" +gen() { + [ -f "$1" ] || openssl rand -hex "$2" | tr -d '\n' > "$1" +} +if [ ! -f postgres_password ]; then + openssl rand -base64 24 | tr -d '\n' > postgres_password +fi +PASS=$(cat postgres_password) +ENC=$(printf '%s' "$PASS" | python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.stdin.read(), safe=""))') +printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$ENC" > database_url +printf 'redis://valkey:6379/0\n' > cache_url +gen broker_password 24 +printf 'requirepass %s\n' "$(cat broker_password)" > broker.conf +printf 'redis://:%s@broker:6379/0\n' "$(cat broker_password)" > broker_url +gen s3_admin_access_key_id 8 +gen s3_admin_secret_access_key 20 +gen s3_access_key_id 8 +gen s3_secret_access_key 20 +BUCKET=$(grep -E '^S3_BUCKET=' "$ROOT/.env" 2>/dev/null | cut -d= -f2 || true) +BUCKET=${BUCKET:-messenger-attachments} +printf '{"identities":[{"name":"admin","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write","List","Tagging"]},{"name":"api","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Read:%s","Write:%s","List:%s"]}]}\n' \ + "$(cat s3_admin_access_key_id)" "$(cat s3_admin_secret_access_key)" "$(cat s3_access_key_id)" "$(cat s3_secret_access_key)" "$BUCKET" "$BUCKET" "$BUCKET" > s3_config.json +chmod 644 ./* +echo "secrets ready in $ROOT/secrets" diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..dc4cf7bc --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +docker compose --env-file .env restart postgres valkey broker s3 +docker compose --env-file .env up -d --wait postgres valkey broker s3 +docker compose --env-file .env restart api lb janitor worker diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..9cee552c --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +dc="docker compose --env-file .env" +old=$($dc ps -q api) +n=$(echo "$old" | wc -w) +$dc build migrate +$dc up -d --no-deps migrate janitor worker +$dc up -d --no-deps --no-recreate --scale api=$((n * 2)) api +$dc up -d --wait --no-deps --no-recreate --scale api=$((n * 2)) api +for c in $old; do + docker stop -t 10 "$c" >/dev/null + docker rm "$c" >/dev/null +done +$dc up -d --wait --no-deps --no-recreate --scale api=$n api diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..1283ec1b --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?}" +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +id=$(docker compose --env-file .env ps -q api | while read -r c; do + case "$c" in "$TARGET_INSTANCE_ID"*) echo "$c";; esac +done) +[ -n "$id" ] || exit 1 +docker stop -t 10 "$id" >/dev/null diff --git a/tests/attachments.py b/tests/attachments.py new file mode 100644 index 00000000..871480c1 --- /dev/null +++ b/tests/attachments.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +import hashlib +import uuid +from pathlib import Path +from client import Client, arguments, check, run +from image_gen import png, png_header_only +from stack import wait_status + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() +MAX_BYTES = 10 * 1024 * 1024 + + +def upload(c, actor, chat, data, expected, content_type='image/png', filename='file.png'): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, expected, + actor=actor, raw=data, headers={'Content-Type': content_type}, + schema='Attachment' if expected == 202 else None) + + +def post_message(c, actor, chat, ids, expected, text=''): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', expected, actor=actor, + body={'text': text, 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message' if expected == 201 else None) + + +def main(): + args = arguments('Attachment validation and access checks') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'second', 'member_ids': [alice['id'], bob['id']]}, schema='Chat')['json'] + eve_chat = c.call('POST', '/api/v1/chats', 201, actor=eve, body={'title': 'eve', 'member_ids': [eve['id'], bob['id']]}, schema='Chat')['json'] + + upload(c, alice, chat, b'x' * (MAX_BYTES + 1), 413) + upload(c, alice, chat, SAMPLE + b'\0' * MAX_BYTES, 413) + upload(c, alice, chat, png_header_only(8000, 8000), 413) + upload(c, alice, chat, SAMPLE[:len(SAMPLE) // 2], 415) + upload(c, alice, chat, SAMPLE[:-1], 415) + upload(c, alice, chat, b'\x89PNG\r\n\x1a\n' + b'garbage' * 100, 415) + upload(c, alice, chat, b'', 415) + upload(c, alice, chat, b'GIF89a' + b'\0' * 64, 415) + upload(c, alice, chat, SAMPLE, 415, content_type='image/jpeg') + upload(c, alice, chat, SAMPLE, 415, content_type='text/plain') + upload(c, alice, chat, SAMPLE, 415, content_type='application/octet-stream') + + for name in ('..%2Fx.png', 'a%5Cb.png', 'a%00b.png', 'a%0Ab.png', 'a' * 256, ''): + upload(c, alice, chat, SAMPLE, 400, filename=name) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments', 400, actor=alice, raw=SAMPLE, headers={'Content-Type': 'image/png'}) + upload(c, eve, chat, SAMPLE, 404) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=a.png', 401, raw=SAMPLE, headers={'Content-Type': 'image/png'}) + + ok = upload(c, alice, chat, SAMPLE, 202)['json'] + check(ok['sha256'] == hashlib.sha256(SAMPLE).hexdigest(), 'checksum mismatch') + check(ok['filename'] == 'file.png' and ok['status'] == 'queued' and ok['variants'] == [], 'attachment fields') + unicode_name = upload(c, alice, chat, SAMPLE, 202, filename='%D1%84%D0%BE%D1%82%D0%BE.png')['json'] + check(unicode_name['filename'] == 'фото.png', 'unicode filename changed') + + post_message(c, alice, other, [ok['id']], 404) + post_message(c, alice, chat, [str(uuid.uuid4())], 404) + post_message(c, alice, chat, [ok['id'], ok['id']], 400) + post_message(c, alice, chat, ['not-a-uuid'], 400) + post_message(c, alice, chat, [str(uuid.uuid4()) for _ in range(11)], 400) + post_message(c, alice, chat, [], 400) + post_message(c, eve, chat, [ok['id']], 404) + post_message(c, eve, eve_chat, [ok['id']], 404) + + shared = post_message(c, bob, chat, [ok['id'], unicode_name['id']], 201, text='from bob')['json'] + check(shared['attachment_ids'] == [ok['id'], unicode_name['id']], 'attachment order changed') + history = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=alice, schema='MessagePage')['json']['items'] + check(any(m['id'] == shared['id'] and m['attachment_ids'] == shared['attachment_ids'] for m in history), 'history lost attachment_ids') + + for suffix in ('', '/content'): + c.call('GET', '/api/v1/attachments/' + ok['id'] + suffix, 404, actor=eve) + c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()) + suffix, 404, actor=alice) + c.call('GET', '/api/v1/attachments/bad' + suffix, 400, actor=alice) + c.call('GET', '/api/v1/attachments/' + ok['id'] + suffix, 401) + c.call('GET', '/api/v1/attachments/' + ok['id'] + '/content?variant=bogus', 400, actor=bob) + wait_status(c, bob, ok['id'], ('ready',), 60) + download = c.call('GET', '/api/v1/attachments/' + ok['id'] + '/content', actor=bob) + check(download['bytes'] == SAMPLE, 'download differs') + + big = png(1800, 1800, noise=True) + check(len(big) < MAX_BYTES, 'generated image exceeds limit') + large = upload(c, alice, chat, big, 202)['json'] + wait_status(c, bob, large['id'], ('ready',), 60) + check(c.call('GET', '/api/v1/attachments/' + large['id'] + '/content', actor=bob)['bytes'] == big, 'large download differs') + print('PASS: attachment validation, limits and access control') + + +if __name__ == '__main__': + run(main) diff --git a/tests/faults.py b/tests/faults.py new file mode 100755 index 00000000..d4edb607 --- /dev/null +++ b/tests/faults.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +import json +import re +import time +import uuid +from client import Client, Failure, arguments, check, run +from stack import (SAMPLE, broker, compose, job_row, logs, now_iso, psql, restore_api, restore_worker, upload, + variant_count, wait_api, wait_log, wait_status) + + +def job_id_of(att_id): + return psql("select id from jobs where attachment_id='%s'" % att_id) + + +def worker_with(fault, **extra): + compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env=dict({'WORKER_FAULT_POINT': fault}, **extra)) + + +def wait_s3(timeout=90): + deadline = time.monotonic() + timeout + while '(healthy)' not in compose('ps', 's3', '--format', '{{.Status}}').stdout: + check(time.monotonic() < deadline, 's3 did not recover') + time.sleep(1) + + +def ops(*args): + out = compose('run', '--rm', '--no-deps', '-T', 'worker', *args).stdout + return [json.loads(line) for line in out.splitlines() if line.startswith('{')] + + +def crash_after_commit(c, alice, bob, chat): + since = now_iso() + compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': 'after_commit'}) + compose('restart', 'lb') + wait_api(c) + name = 'crash-' + uuid.uuid4().hex[:12] + '.png' + answered = None + try: + answered = upload(c, alice, chat, filename=name) + except Failure: + pass + check(answered is None, 'api answered 202 despite injected crash') + wait_log('api', since, '"point":"after_commit"', 20) + att_id = psql("select id from attachments where filename='%s'" % name) + check(att_id, 'committed attachment not found') + restore_api(c) + ready = wait_status(c, bob, att_id, ('ready', 'failed'), 60) + check(ready['status'] == 'ready' and job_row(att_id)[:2] == ('ready', 1), 'outbox did not recover the job') + published = psql("select count(*) from outbox where job_id='%s' and published_at is not null" % job_id_of(att_id)) + print('1) api crashed after COMMIT before 202/publish: attachment=%s recovered by outbox relay (published rows=%s) -> ready attempt=1' + % (att_id, published)) + + +def crash_after_store(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + since = now_iso() + worker_with('after_store', JOB_TIMEOUT_SECONDS='10') + wait_log('worker', since, '"point":"after_store"', 60) + status, attempt, _ = job_row(att['id']) + check(status == 'processing' and attempt == 1, 'job must stay processing under expired-to-be lease') + since = now_iso() + worker_with('', JOB_TIMEOUT_SECONDS='10') + wait_log('worker', since, '"lease expired","job_id":"%s"' % job_id_of(att['id']), 90) + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + status, attempt, _ = job_row(att['id']) + check(ready['status'] == 'ready' and status == 'ready' and attempt >= 2, 'job did not finish after lease expiry') + check(variant_count(att['id']) == 3 and len(ready['variants']) == 3, 'retry duplicated variants') + thumb = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0] + body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)['bytes'] + check(len(body) == thumb['size_bytes'], 'thumbnail differs from metadata') + restore_worker() + print('2) worker crashed after S3 write before COMMIT: lease expired, reaper requeued, attempt=%d ready, variants=3' % attempt) + + +def crash_before_ack(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + job_id = job_id_of(att['id']) + since = now_iso() + worker_with('before_ack') + wait_log('worker', since, '"point":"before_ack"', 60) + wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 60) + status, attempt, _ = job_row(att['id']) + check(status == 'ready' and attempt == 1 and variant_count(att['id']) == 3, 'redelivery after commit changed result') + check(broker('XPENDING', 'attachments.jobs', 'workers').splitlines()[0] == '0', 'redelivered message not acked') + check(wait_status(c, bob, att['id'], ('ready',), 10)['status'] == 'ready', 'attachment not ready') + restore_worker() + print('3) worker crashed after COMMIT before ack: message redelivered after restart, skipped as no-op, attempt=1, variants=3') + + +def retries_and_replay(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + job_id = job_id_of(att['id']) + compose('stop', 's3') + since = now_iso() + try: + compose('up', '-d', '--no-deps', 'worker') + check('(healthy)' not in compose('ps', '-a', 's3', '--format', '{{.Status}}').stdout, 's3 must stay stopped') + failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 120) + check(failed['status'] == 'failed' and failed['error_code'] == 'retry_exhausted', 'expected retry_exhausted') + text = logs('worker', since) + delays = [int(d) for d in re.findall(r'"job retry scheduled","job_id":"%s".*?"delay_ms":(\d+)' % job_id, text)] + check(len(delays) == 4, 'expected 4 scheduled retries, got %d' % len(delays)) + for n, d in enumerate(delays): + base = 1000 * 2 ** n + check(base // 2 <= d <= base, 'delay %d outside jittered window for retry %d' % (d, n + 1)) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, body={'text': '', 'attachment_ids': [att['id']]}, + headers={'Idempotency-Key': uuid.uuid4().hex}) + check(any(x['attachment_id'] == att['id'] for x in ops('failed')), 'failed registry misses job') + print('4) s3 down: transient errors retried with backoff+jitter delays_ms=%s, then failed retry_exhausted, listed in registry' % delays) + finally: + compose('start', 's3') + wait_s3() + check(ops('replay', att['id'])[0]['result'] == 'queued', 'replay did not queue job') + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 120) + check(ready['status'] == 'ready' and variant_count(att['id']) == 3, 'replayed job did not finish') + check(ops('replay', att['id'])[0]['result'] == 'skipped_ready', 'replay of ready job must be a no-op') + check(psql("select count(*) from job_replays where job_id='%s'" % job_id) == '1', 'replay audit missing') + check(variant_count(att['id']) == 3, 'second replay created new effects') + print(' replay: failed -> queued -> ready (audit rows=1); replay of ready job skipped, variants=3') + + +def permanent_error(c, alice, bob, chat): + data = bytearray(SAMPLE) + mid = len(data) // 2 + data[mid:mid + 64] = b'\x00' * 64 + att = upload(c, alice, chat, bytes(data))['json'] + failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 60) + status, attempt, _ = job_row(att['id']) + check(failed['status'] == 'failed' and failed['error_code'] == 'invalid_image' and attempt == 1, 'corrupt image must fail without retries') + check(failed['variants'] == [], 'failed attachment has variants') + print('5) corrupted IDAT (passes header/tail check in API): failed invalid_image on attempt 1, no retries') + + +def main(): + args = arguments('Fault injection: COMMIT/publish gap, crash before COMMIT and before ack, retries, DLQ replay') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + try: + crash_after_commit(c, alice, bob, chat) + crash_after_store(c, alice, bob, chat) + crash_before_ack(c, alice, bob, chat) + retries_and_replay(c, alice, bob, chat) + permanent_error(c, alice, bob, chat) + finally: + compose('up', '-d', '--no-deps', 's3') + restore_worker() + if 'after_commit' in compose('exec', '-T', 'api', 'printenv', 'FAULT_POINT', check_rc=False).stdout: + restore_api(c) + print('PASS: outbox recovery, lease recovery, redelivery after commit, backoff with jitter, retry_exhausted, replay, permanent error') + + +if __name__ == '__main__': + run(main) diff --git a/tests/idempotency.py b/tests/idempotency.py new file mode 100644 index 00000000..951d4385 --- /dev/null +++ b/tests/idempotency.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import threading +import uuid +from client import Client, Failure, arguments, check, run + + +def main(): + args = arguments('Concurrent Idempotency-Key check through the load balancer') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/messages' + parallel = 32 + key = 'idem-' + uuid.uuid4().hex + barrier = threading.Barrier(parallel) + + def send(_): + local = Client(args.base_url, args.ca_file) + barrier.wait() + return local.call('POST', path, 201, actor=alice, body={'text': 'same'}, headers={'Idempotency-Key': key}, schema='Message') + + with ThreadPoolExecutor(max_workers=parallel) as pool: + results = list(pool.map(send, range(parallel))) + ids = {r['json']['id'] for r in results} + instances = {r['headers']['X-Instance-Id'] for r in results} + check(len(ids) == 1, 'parallel requests produced %d messages' % len(ids)) + check(len(instances) >= 2, 'requests were served by one instance only') + check(len({r['json']['created_at'] for r in results}) == 1, 'created_at differs between replays') + + listing = c.call('GET', path, actor=alice, schema='MessagePage')['json']['items'] + check(sum(1 for m in listing if m['text'] == 'same') == 1, 'duplicate message in history') + + c.call('POST', path, 409, actor=alice, body={'text': 'other'}, headers={'Idempotency-Key': key}) + c.call('POST', path, 400, actor=alice, body={'text': 'x'}) + c.call('POST', path, 400, actor=alice, body={'text': 'x'}, headers={'Idempotency-Key': 'bad key!'}) + other = c.call('POST', path, 201, actor=bob, body={'text': 'same'}, headers={'Idempotency-Key': key}, schema='Message')['json'] + check(other['id'] not in ids, 'key must be scoped per user') + print('PASS: %d parallel requests, 1 message, %d instances' % (parallel, len(instances))) + + +if __name__ == '__main__': + run(main) diff --git a/tests/image_gen.py b/tests/image_gen.py new file mode 100644 index 00000000..aeb7de0f --- /dev/null +++ b/tests/image_gen.py @@ -0,0 +1,23 @@ +import os +import struct +import zlib + + +def chunk(kind, data): + body = kind + data + return struct.pack('>I', len(data)) + body + struct.pack('>I', zlib.crc32(body) & 0xffffffff) + + +def png(width, height, noise=False): + header = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + raw = bytearray() + row = width * 3 + 1 + for _ in range(height): + raw += b'\x00' + (os.urandom(row - 1) if noise else bytes(row - 1)) + data = zlib.compress(bytes(raw), 0 if noise else 6) + return header + chunk(b'IDAT', data) + chunk(b'IEND', b'') + + +def png_header_only(width, height): + header = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + return header + chunk(b'IDAT', zlib.compress(b'\x00')) + chunk(b'IEND', b'') diff --git a/tests/queue_recovery.py b/tests/queue_recovery.py new file mode 100755 index 00000000..0c420d8a --- /dev/null +++ b/tests/queue_recovery.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +import json +import time +from client import Client, arguments, check, run +from image_probe import jpeg_size +from stack import broker, compose, job_row, logs, now_iso, psql, upload, variant_count, wait_api, wait_log, wait_status + + +def stream_has(job_id): + return job_id in broker('XRANGE', 'attachments.jobs', '-', '+') + + +def main(): + args = arguments('Worker stop, API/broker restart and redelivery scenario') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + + compose('stop', 'worker') + try: + att = upload(c, alice, chat)['json'] + check(att['status'] == 'queued' and att['variants'] == [], 'upload must return queued snapshot') + msg = c.message(alice, chat, '', attachments=[att['id']])['json'] + check(msg['attachment_ids'] == [att['id']], 'message lost queued attachment') + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 409, actor=bob) + c.message(bob, chat, 'text works while worker is down') + job_id = psql("select id from jobs where attachment_id='%s'" % att['id']) + deadline = time.monotonic() + 10 + while not stream_has(job_id): + check(time.monotonic() < deadline, 'job was not published to broker') + time.sleep(0.5) + check(psql("select count(*) from outbox where job_id='%s' and published_at is null" % job_id) == '0', 'outbox not marked') + print('accepted: attachment=%s job=%s status=queued, job in stream, worker stopped' % (att['id'], job_id)) + + compose('restart', 'broker', 'api') + compose('up', '-d', '--wait', '--no-deps', 'broker', 'api') + compose('restart', 'lb') + wait_api(c) + check(stream_has(job_id), 'job lost after broker restart') + check(c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status'] == 'queued', 'status changed without worker') + print('restarted broker and api: job still in stream, attachment still queued') + finally: + compose('start', 'worker') + + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + check(ready['status'] == 'ready', 'job did not finish after worker start') + thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob) + check(jpeg_size(thumb['bytes']) == (256, 160), 'thumbnail size') + check(job_row(att['id'])[:2] == ('ready', 1) and variant_count(att['id']) == 3, 'unexpected job state or variants') + print('worker started: attachment ready, attempt=1, variants=3') + + since = now_iso() + before = psql("select updated_at from attachments where id='%s'" % att['id']) + payload = json.dumps({'schema_version': 1, 'job_id': job_id, 'type': 'attachment.process.v1', 'attachment_id': att['id'], + 'pipeline_version': 1, 'attempt': 1, 'created_at': ready['created_at']}) + broker('XADD', 'attachments.jobs', '*', 'job', payload) + wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 30) + time.sleep(1) + check(psql("select updated_at from attachments where id='%s'" % att['id']) == before, 'duplicate delivery changed attachment') + check(variant_count(att['id']) == 3 and job_row(att['id'])[:2] == ('ready', 1), 'duplicate delivery created new effects') + pending = broker('XPENDING', 'attachments.jobs', 'workers') + check(pending.splitlines()[0] == '0', 'duplicate message was not acked') + print('duplicate delivery: skipped as no-op, acked, variants=3, updated_at unchanged') + print('PASS: worker stop, broker/api restart, redelivery idempotency') + + +if __name__ == '__main__': + run(main) diff --git a/tests/s3_checks.py b/tests/s3_checks.py new file mode 100644 index 00000000..25413909 --- /dev/null +++ b/tests/s3_checks.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +import hashlib +import subprocess +import time +import uuid +from pathlib import Path +from client import Client, Failure, arguments, check, run +from stack import wait_status + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() +BUCKET = 'messenger-attachments' + + +def compose(*args, check_rc=True, stdin=None): + return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=check_rc, + capture_output=True, text=True, input=stdin) + + +def secret(name): + return (ROOT / 'secrets' / name).read_text().strip() + + +def s3_status(method, path, user=None, data=None): + cmd = ['exec', '-T', 's3', 'curl', '-s', '-o', '/dev/null', '-w', '%{http_code}', '-X', method] + if user: + cmd += ['--aws-sigv4', 'aws:amz:us-east-1:s3', '--user', user[0] + ':' + user[1]] + if data is not None: + cmd += ['--data-binary', data] + cmd.append('http://127.0.0.1:9000' + path) + return compose(*cmd).stdout.strip() + + +def upload(c, actor, chat, expected): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=a.png', expected, actor=actor, + raw=SAMPLE, headers={'Content-Type': 'image/png'}, schema='Attachment' if expected == 202 else None) + + +def wait_api(c): + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + try: + c.call('GET', '/health/ready') + return + except Failure: + time.sleep(1) + raise Failure('api did not recover') + + +def wait_s3(timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if compose('ps', 's3', '--format', '{{.Status}}').stdout.count('(healthy)'): + return + time.sleep(1) + raise Failure('s3 did not recover') + + +def object_key(attachment_id): + out = compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', + "select object_key from attachments where id='%s'" % attachment_id).stdout.strip() + check(out.startswith('attachments/'), 'object key not found') + return out + + +def main(): + args = arguments('S3 privacy, outage and orphan cleanup checks') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + att = upload(c, alice, chat, 202)['json'] + wait_status(c, bob, att['id'], ('ready',), 60) + key = object_key(att['id']) + check(key != 'attachments/' + att['filename'] and att['filename'] not in key, 'object key derived from filename') + + api = (secret('s3_access_key_id'), secret('s3_secret_access_key')) + admin = (secret('s3_admin_access_key_id'), secret('s3_admin_secret_access_key')) + path = '/' + BUCKET + '/' + key + check(s3_status('GET', path) == '403', 'anonymous GET must be denied') + check(s3_status('GET', '/' + BUCKET + '/') == '403', 'anonymous listing must be denied') + check(s3_status('PUT', path + '-anon', data='x') == '403', 'anonymous PUT must be denied') + check(s3_status('GET', path, ('wrong', 'wrong')) == '403', 'wrong credentials must be denied') + check(s3_status('GET', path, api) == '200', 'api credentials cannot read') + check(s3_status('PUT', '/other-bucket-' + uuid.uuid4().hex[:8], api) == '403', 'api credentials can create buckets') + ports = compose('ps', '--format', '{{.Service}} {{.Ports}}').stdout + published = [line.split()[0] for line in ports.splitlines() if '->' in line] + check(published == ['lb'], 'only lb may publish ports, found %s' % published) + + compose('stop', 's3') + try: + upload(c, alice, chat, 503) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob) + c.call('GET', '/api/v1/attachments/' + att['id'], 200, actor=bob, schema='Attachment') + c.message(alice, chat, 'text works while s3 is down') + finally: + compose('start', 's3') + wait_s3() + wait_api(c) + deadline = time.monotonic() + 60 + while True: + try: + body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] + check(body == SAMPLE, 'content differs after s3 restart') + break + except Failure: + check(time.monotonic() < deadline, 'download did not recover') + time.sleep(1) + upload(c, alice, chat, 202) + + orphan = '/' + BUCKET + '/attachments/' + uuid.uuid4().hex + check(s3_status('PUT', orphan, admin, 'orphan') == '200', 'cannot create orphan') + compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=3600', 'janitor', 'cleanup') + check(s3_status('GET', orphan, admin) == '200', 'fresh object was removed inside grace period') + time.sleep(1) + compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=0', 'janitor', 'cleanup') + check(s3_status('GET', orphan, admin) == '404', 'orphan was not removed') + check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == SAMPLE, 'referenced object was removed') + + check(s3_status('GET', '/' + BUCKET + '/staging/' + att['id'], admin) == '404', 'staging kept after ready') + stale = '/' + BUCKET + '/staging/' + str(uuid.uuid4()) + check(s3_status('PUT', stale, admin, 'stale') == '200', 'cannot create stale staging object') + compose('stop', 'worker') + try: + queued = upload(c, alice, chat, 202)['json'] + compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=0', 'janitor', 'cleanup') + check(s3_status('GET', '/' + BUCKET + '/staging/' + queued['id'], admin) == '200', 'staging of queued job was removed') + check(s3_status('GET', stale, admin) == '404', 'unreferenced staging object was not removed') + finally: + compose('up', '-d', '--no-deps', 'worker') + check(wait_status(c, bob, queued['id'], ('ready', 'failed'), 60)['status'] == 'ready', 'queued job lost staging') + print('PASS: private bucket, limited credentials, s3 outage, orphan and staging cleanup') + + +if __name__ == '__main__': + run(main) diff --git a/tests/scenario.py b/tests/scenario.py new file mode 100644 index 00000000..f7ddd700 --- /dev/null +++ b/tests/scenario.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import json +import subprocess +import threading +import time +import uuid +from pathlib import Path +from client import Client, Failure, arguments, check, run + +ROOT = Path(__file__).resolve().parent.parent + + +def sh(*cmd): + return subprocess.run(cmd, cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip() + + +def api_containers(): + out = sh('docker', 'compose', '--env-file', '.env', 'ps', '-q', 'api') + return [x[:12] for x in out.split()] + + +def wait_healthy(count, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + out = sh('docker', 'compose', '--env-file', '.env', 'ps', 'api', '--format', '{{.Status}}') + if out.count('(healthy)') >= count: + return + time.sleep(1) + raise Failure('replicas did not become healthy') + + +class Load: + def __init__(self, base, ca, concurrency): + c = Client(base, ca) + self.alice, _, _, self.chat = c.bootstrap() + self.base, self.ca, self.n = base, ca, concurrency + self.events = [] + self.lock = threading.Lock() + self.stop = threading.Event() + self.t0 = time.monotonic() + + def worker(self): + c = Client(self.base, self.ca, timeout=3) + while not self.stop.is_set(): + key = uuid.uuid4().hex + first_error = None + while not self.stop.is_set(): + try: + r = c.message(self.alice, self.chat, 'k-' + key, key=key) + t = time.monotonic() - self.t0 + with self.lock: + self.events.append((t, True, r['headers']['X-Instance-Id'], first_error)) + break + except Failure: + t = time.monotonic() - self.t0 + first_error = first_error or t + with self.lock: + self.events.append((t, False, None, None)) + time.sleep(0.05) + + def start(self): + self.pool = ThreadPoolExecutor(max_workers=self.n) + self.futures = [self.pool.submit(self.worker) for _ in range(self.n)] + + def finish(self): + self.stop.set() + self.pool.shutdown(wait=True) + + def summary(self, begin, end): + ev = [e for e in self.events if begin <= e[0] < end] + ok = [e for e in ev if e[1]] + bad = [e for e in ev if not e[1]] + span = max(end - begin, 1e-9) + return { + 'window_seconds': round(span, 2), + 'successful_requests': len(ok), + 'failed_attempts': len(bad), + 'successful_rps': round(len(ok) / span, 1), + 'last_error_offset_seconds': round(max(e[0] for e in bad) - begin, 2) if bad else None, + 'instances': sorted({e[2] for e in ok}), + } + + def instances_between(self, begin, end=float('inf')): + return sorted({e[2] for e in self.events if e[1] and begin <= e[0] < end}) + + +def failover_each(args): + ids = api_containers() + check(len(ids) >= 2, 'need at least two replicas') + load = Load(args.base_url, args.ca_file, args.concurrency) + load.start() + time.sleep(args.warmup) + results = [] + base = load.summary(2, args.warmup) + for target in ids: + before = time.monotonic() - load.t0 + sh('docker', 'stop', '-t', '10', target) + stopped = time.monotonic() - load.t0 + time.sleep(args.hold) + after_stop = time.monotonic() - load.t0 + window = load.summary(before, after_stop) + sh('docker', 'start', target) + wait_healthy(len(ids)) + restored = time.monotonic() - load.t0 + time.sleep(args.warmup) + results.append({ + 'stopped_replica': target, + 'during_failure': window, + 'seen_stopped_replica_after_stop': target in load.instances_between(stopped + 0.5, after_stop), + 'stop_command_seconds': round(stopped - before, 2), + 'restore_seconds': round(restored - after_stop, 2), + 'after_restore': load.summary(restored + 1, restored + args.warmup), + }) + load.finish() + return {'baseline': base, 'replicas': results} + + +def rolling(args): + old = api_containers() + check(len(old) >= 2, 'need at least two replicas') + n = len(old) + load = Load(args.base_url, args.ca_file, args.concurrency) + load.start() + time.sleep(args.warmup) + begin = time.monotonic() - load.t0 + sh('scripts/rolling-update.sh') + end = time.monotonic() - load.t0 + time.sleep(args.warmup) + load.finish() + return { + 'old_replicas': old, + 'new_replicas': api_containers(), + 'update_window': load.summary(begin, end), + 'old_replicas_served_after_update': [i for i in load.instances_between(end) if i in old], + } + + +def main(): + def configure(p): + p.add_argument('--mode', choices=['failover-each', 'rolling'], required=True) + p.add_argument('--concurrency', type=int, default=8) + p.add_argument('--warmup', type=float, default=8) + p.add_argument('--hold', type=float, default=12) + p.add_argument('--output') + args = arguments('Failure and rolling update under load', configure) + report = failover_each(args) if args.mode == 'failover-each' else rolling(args) + text = json.dumps(report, indent=2) + if args.output: + Path(args.output).write_text(text + '\n') + print(text) + + +if __name__ == '__main__': + run(main) diff --git a/tests/session_revocation.py b/tests/session_revocation.py new file mode 100644 index 00000000..2e2a80de --- /dev/null +++ b/tests/session_revocation.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import time +from client import Client, arguments, check, run +from stack import compose + + +def wait_valkey(timeout=60): + deadline = time.monotonic() + timeout + while '(healthy)' not in compose('ps', 'valkey', '--format', '{{.Status}}').stdout: + check(time.monotonic() < deadline, 'valkey did not recover') + time.sleep(1) + + +def main(): + args = arguments('Session revocation must not depend on the cache') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + c.call('GET', '/api/v1/chats', 200, actor=alice) + compose('stop', 'valkey') + try: + c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice) + finally: + compose('start', 'valkey') + wait_valkey() + c.call('GET', '/api/v1/chats', 401, actor=alice) + c.call('GET', '/api/v1/chats', 200, actor=bob) + print('PASS: token revoked while valkey was stopped stays revoked after valkey restart (401)') + + +if __name__ == '__main__': + run(main) diff --git a/tests/stack.py b/tests/stack.py new file mode 100644 index 00000000..f0b645f6 --- /dev/null +++ b/tests/stack.py @@ -0,0 +1,92 @@ +import os +import subprocess +import time +import uuid +from pathlib import Path +from client import Failure, check + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() + + +def compose(*args, env=None, check_rc=True): + full = dict(os.environ, **(env or {})) + return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=check_rc, + capture_output=True, text=True, env=full) + + +def psql(sql): + return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', sql).stdout.strip() + + +def broker(*args): + cmd = 'valkey-cli --no-auth-warning -a "$(sed -n "s/^requirepass //p" /run/secrets/broker_conf)" ' + ' '.join("'%s'" % a for a in args) + return compose('exec', '-T', 'broker', 'sh', '-c', cmd).stdout.strip() + + +def logs(service, since): + return compose('logs', '--no-color', '--since', since, service).stdout + + +def now_iso(): + return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) + + +def upload(c, actor, chat, data=SAMPLE, content_type='image/png', filename=None, expected=202): + filename = filename or 'f-' + uuid.uuid4().hex[:12] + '.png' + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, expected, actor=actor, + raw=data, headers={'Content-Type': content_type}, schema='Attachment' if expected == 202 else None) + + +def wait_status(c, actor, att_id, statuses, timeout=120): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + att = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json'] + if att['status'] in statuses: + return att + except Failure: + pass + time.sleep(0.5) + raise Failure('attachment %s did not reach %s' % (att_id, statuses)) + + +def wait_api(c, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + c.call('GET', '/health/ready') + return + except Failure: + time.sleep(1) + raise Failure('api did not become ready') + + +def wait_log(service, since, needle, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if needle in logs(service, since): + return + time.sleep(1) + raise Failure('%s did not log %r' % (service, needle)) + + +def job_row(att_id): + out = psql("select status, attempt, coalesce(last_error, '') from jobs where attachment_id='%s'" % att_id) + check(out, 'job not found') + status, attempt, error = out.split('|', 2) + return status, int(attempt), error + + +def variant_count(att_id): + return int(psql("select count(*) from attachment_variants where attachment_id='%s'" % att_id)) + + +def restore_worker(): + compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'WORKER_FAULT_POINT': ''}) + + +def restore_api(c): + compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': ''}) + compose('restart', 'lb') + wait_api(c) diff --git a/tests/upload_memory.py b/tests/upload_memory.py new file mode 100644 index 00000000..aa88929f --- /dev/null +++ b/tests/upload_memory.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import re +import subprocess +import threading +import time +from pathlib import Path +from client import Client, Failure, arguments, check, run +from image_gen import png + +ROOT = Path(__file__).resolve().parent.parent +UNITS = {'B': 1, 'KiB': 1024, 'MiB': 1024 ** 2, 'GiB': 1024 ** 3} + + +def to_mib(text): + value, unit = re.match(r'([\d.]+)\s*([A-Za-z]+)', text).groups() + return float(value) * UNITS[unit] / 1024 ** 2 + + +def sample(stop, peaks): + proc = subprocess.Popen(['docker', 'stats', '--format', '{{.Name}} {{.MemUsage}}'], cwd=ROOT, + stdout=subprocess.PIPE, text=True) + try: + for line in proc.stdout: + line = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line).strip() + m = re.match(r'(\S+-api-\d+)\s+(\S+ ?\S+) /', line) + if m: + peaks[m.group(1)] = max(peaks.get(m.group(1), 0), to_mib(m.group(2))) + if stop.is_set(): + break + finally: + proc.terminate() + + +def main(): + def configure(p): + p.add_argument('--concurrency', type=int, default=12) + p.add_argument('--rounds', type=int, default=3) + p.add_argument('--output') + args = arguments('Concurrent upload memory experiment', configure) + check(args.concurrency >= 10, 'use at least 10 concurrent uploads') + c = Client(args.base_url, args.ca_file, timeout=60) + alice, bob, eve, chat = c.bootstrap() + data = png(1800, 1800, noise=True) + digest = hashlib.sha256(data).hexdigest() + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=load.png' + + idle, stop = {}, threading.Event() + t = threading.Thread(target=sample, args=(stop, idle)) + t.start() + time.sleep(4) + stop.set() + t.join() + + peaks, stop = {}, threading.Event() + t = threading.Thread(target=sample, args=(stop, peaks)) + t.start() + time.sleep(2) + barrier = threading.Barrier(args.concurrency) + results = [] + + def worker(_): + local = Client(args.base_url, args.ca_file, timeout=60) + barrier.wait() + out = [] + for _ in range(args.rounds): + try: + r = local.call('POST', path, 202, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment') + out.append(r['json']['sha256'] == digest) + except Failure: + out.append(False) + return out + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + for out in pool.map(worker, range(args.concurrency)): + results.extend(out) + elapsed = time.monotonic() - started + time.sleep(2) + stop.set() + t.join() + limit = 192 + report = { + 'file_bytes': len(data), + 'concurrent_uploads': args.concurrency, + 'rounds': args.rounds, + 'uploads_ok': sum(results), + 'uploads_failed': len(results) - sum(results), + 'elapsed_seconds': round(elapsed, 2), + 'api_memory_limit_mib': limit, + 'idle_memory_mib': {k: round(v, 1) for k, v in sorted(idle.items())}, + 'peak_memory_mib': {k: round(v, 1) for k, v in sorted(peaks.items())}, + 'buffering_all_files_would_need_mib': round(args.concurrency * len(data) / 1024 ** 2, 1), + } + text = json.dumps(report, indent=2) + if args.output: + Path(args.output).write_text(text + '\n') + print(text) + check(report['uploads_failed'] == 0, 'some uploads failed') + check(max(peaks.values()) < limit, 'peak memory reached the limit') + + +if __name__ == '__main__': + run(main) diff --git a/tests/variants.py b/tests/variants.py new file mode 100755 index 00000000..4c009050 --- /dev/null +++ b/tests/variants.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +import hashlib +import io +from PIL import Image +from client import Client, arguments, check, run +from stack import SAMPLE, upload, wait_status + +EXPECTED = {'thumbnail': ('image/jpeg', 'JPEG'), 'crop': ('image/jpeg', 'JPEG'), 'optimized': ('image/webp', 'WEBP')} + + +def encode(img, fmt, **kw): + buf = io.BytesIO() + img.save(buf, fmt, **kw) + return buf.getvalue() + + +def halves(w, h): + img = Image.new('RGB', (w, h), (0, 0, 255)) + img.paste((255, 0, 0), (0, 0, w // 2, h)) + return img + + +def process(c, alice, bob, chat, data, content_type='image/png'): + att = upload(c, alice, chat, data, content_type)['json'] + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + check(ready['status'] == 'ready' and ready['error_code'] is None, 'attachment failed: %s' % ready['error_code']) + check(sorted(v['name'] for v in ready['variants']) == sorted(EXPECTED), 'unexpected variant set') + original = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] + check(original == data and ready['sha256'] == hashlib.sha256(data).hexdigest(), 'original changed') + out = {} + for v in ready['variants']: + r = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + v['name'], actor=bob) + body = r['bytes'] + check(r['headers'].get_content_type() == v['content_type'] == EXPECTED[v['name']][0], v['name'] + ' content type') + check(len(body) == v['size_bytes'] and hashlib.sha256(body).hexdigest() == v['sha256'], v['name'] + ' checksum') + img = Image.open(io.BytesIO(body)) + check(img.format == EXPECTED[v['name']][1] and img.size == (v['width'], v['height']), v['name'] + ' real size/format') + check('exif' not in img.info, v['name'] + ' keeps EXIF') + img.load() + out[v['name']] = img.convert('RGB') + return att, out + + +def main(): + args = arguments('Thumbnail, crop and optimized variant rules') + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + + att, v = process(c, alice, bob, chat, SAMPLE) + check(v['thumbnail'].size == (256, 160) and v['crop'].size == (128, 128) and v['optimized'].size == (320, 200), 'fixture sizes') + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=bogus', 400, actor=bob) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 404, actor=eve) + + _, v = process(c, alice, bob, chat, encode(halves(2000, 1000), 'PNG')) + check(v['thumbnail'].size == (256, 128) and v['optimized'].size == (1280, 640) and v['crop'].size == (128, 128), 'large image sizes') + + _, v = process(c, alice, bob, chat, encode(halves(100, 50), 'PNG')) + check(v['thumbnail'].size == (100, 50) and v['optimized'].size == (100, 50), 'small image must not be upscaled') + check(v['crop'].size == (128, 128), 'crop must be 128x128') + + _, v = process(c, alice, bob, chat, encode(Image.new('RGBA', (300, 300), (0, 0, 0, 0)), 'PNG')) + for name, img in v.items(): + check(min(img.getpixel((img.width // 2, img.height // 2))) >= 250, name + ': alpha not composited on white') + + exif = Image.Exif() + exif[0x0112] = 6 + _, v = process(c, alice, bob, chat, encode(halves(300, 200), 'JPEG', quality=95, exif=exif.tobytes()), 'image/jpeg') + t = v['thumbnail'] + check(t.size == (170, 256) and v['optimized'].size == (200, 300), 'EXIF orientation not applied to dimensions') + top, bottom = t.getpixel((85, 20)), t.getpixel((85, 236)) + check(top[0] > 200 and top[2] < 60 and bottom[2] > 200 and bottom[0] < 60, 'EXIF rotation direction wrong') + print('PASS: thumbnail/crop/optimized sizes, formats, checksums, no upscale, alpha on white, EXIF orientation, no EXIF in variants') + + +if __name__ == '__main__': + run(main) diff --git a/tests/worker_load.py b/tests/worker_load.py new file mode 100755 index 00000000..97bdfd7e --- /dev/null +++ b/tests/worker_load.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +import json +import subprocess +import threading +import time +from pathlib import Path +from client import Client, arguments, check, run +from image_gen import png +from stack import SAMPLE, compose, job_row, psql, restore_worker, upload, wait_status + +LIMIT_MIB = 768 + + +def sample(stop, state): + while not stop.is_set(): + out = compose('exec', '-T', 'worker', 'grep', 'VmRSS', '/proc/1/status', check_rc=False).stdout.split() + if len(out) > 1: + state['rss'] = max(state['rss'], int(out[1]) / 1024) + n = psql("select count(*) from jobs where status = 'processing'") + state['processing'] = max(state['processing'], int(n or 0)) + time.sleep(0.2) + + +def main(): + def configure(p): + p.add_argument('--huge', type=int, default=6) + p.add_argument('--small', type=int, default=20) + p.add_argument('--output') + args = arguments('Worker concurrency, memory and time limits', configure) + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + concurrency = int(compose('exec', '-T', 'worker', 'printenv', 'WORKER_CONCURRENCY').stdout.strip()) + huge = png(6320, 6320) + + state, stop = {'rss': 0, 'processing': 0}, threading.Event() + t = threading.Thread(target=sample, args=(stop, state)) + t.start() + started = time.monotonic() + ids = [upload(c, alice, chat, huge)['json']['id'] for _ in range(args.huge)] + ids += [upload(c, alice, chat, SAMPLE)['json']['id'] for _ in range(args.small)] + accepted = time.monotonic() - started + results = [wait_status(c, bob, i, ('ready', 'failed'), 300)['status'] for i in ids] + elapsed = time.monotonic() - started + stop.set() + t.join() + container = compose('ps', '-q', 'worker').stdout.strip() + oom = subprocess.run(['docker', 'inspect', '-f', '{{.RestartCount}} {{.State.OOMKilled}}', container], + capture_output=True, text=True).stdout.split() + + compose('stop', 'worker') + slow = upload(c, alice, chat, huge)['json'] + compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'JOB_TIMEOUT_SECONDS': '1'}) + timed = wait_status(c, bob, slow['id'], ('ready', 'failed'), 120) + _, attempts, last_error = job_row(slow['id']) + restore_worker() + + report = { + 'huge_images': {'count': args.huge, 'pixels': 6320 * 6320, 'bytes': len(huge)}, + 'small_images': args.small, + 'worker_concurrency': concurrency, + 'max_processing_jobs_observed': state['processing'], + 'all_ready': results.count('ready') == len(results), + 'accept_seconds': round(accepted, 2), + 'all_ready_seconds': round(elapsed, 1), + 'worker_peak_rss_mib': round(state['rss'], 1), + 'worker_memory_limit_mib': LIMIT_MIB, + 'worker_restarts_oom': oom, + 'timeout_case': {'job_timeout_seconds': 1, 'status': timed['status'], 'error_code': timed['error_code'], + 'attempts': attempts, 'last_error': last_error}, + } + text = json.dumps(report, indent=2) + if args.output: + Path(args.output).write_text(text + '\n') + print(text) + check(report['all_ready'], 'some jobs failed under load') + check(state['processing'] <= concurrency, 'more jobs processing than WORKER_CONCURRENCY') + check(state['rss'] < LIMIT_MIB and oom == ['0', 'false'], 'worker reached memory limit') + check(timed['status'] == 'failed' and timed['error_code'] == 'retry_exhausted' and 'deadline' in last_error, + 'timeout must be a bounded transient failure') + print('PASS: bounded worker concurrency, memory below limit, job timeout enforced') + + +if __name__ == '__main__': + run(main) diff --git a/worker.go b/worker.go new file mode 100644 index 00000000..d8cfe749 --- /dev/null +++ b/worker.go @@ -0,0 +1,448 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "math/rand/v2" + "os" + "os/signal" + "strings" + "sync" + "syscall" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" +) + +var errSource = errors.New("staging object missing or corrupted") + +type Worker struct { + db *pgxpool.Pool + store *Store + broker *Broker + log *slog.Logger + owner string + timeout time.Duration + lease time.Duration + backoffBase time.Duration + backoffMax time.Duration + maxBytes int64 + maxPixels int64 + fault string +} + +type claimedJob struct { + id, attachmentID, owner string + attempt int +} + +type source struct { + stagingKey, contentType, sha256 string + size int64 +} + +func finalKey(attachmentID string) string { + return objectPrefix + attachmentID + "/original" +} + +func variantKey(attachmentID string, spec variantSpec) string { + return fmt.Sprintf("%s%s/v%d/%s.%s", objectPrefix, attachmentID, pipelineVersion, spec.name, spec.ext) +} + +func (w *Worker) inject(point string) { + if w.fault == point { + w.log.Error("fault injected", "point", point) + os.Exit(3) + } +} + +func (w *Worker) backoff(attempt int) time.Duration { + d := w.backoffBase << (attempt - 1) + if d <= 0 || d > w.backoffMax { + d = w.backoffMax + } + return d/2 + rand.N(d/2+1) +} + +func (w *Worker) claim(ctx context.Context, jobID string) (*claimedJob, error) { + tx, err := w.db.Begin(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback(ctx) + job := &claimedJob{id: jobID, owner: w.owner} + err = tx.QueryRow(ctx, ` + update jobs set status = 'processing', lease_owner = $2, + lease_until = now() + make_interval(secs => $3), updated_at = now() + where id = $1::uuid and status = 'queued' and next_attempt_at <= now() + returning attachment_id::text, attempt`, jobID, w.owner, w.lease.Seconds()).Scan(&job.attachmentID, &job.attempt) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if _, err := tx.Exec(ctx, `update attachments set status = 'processing', updated_at = now() where id = $1::uuid`, job.attachmentID); err != nil { + return nil, err + } + return job, tx.Commit(ctx) +} + +func (w *Worker) fetch(ctx context.Context, src source) ([]byte, error) { + obj, info, err := w.store.Get(ctx, src.stagingKey) + if err != nil { + if isNoSuchKey(err) { + return nil, errSource + } + return nil, err + } + defer obj.Close() + if info.Size != src.size || info.Size > w.maxBytes { + return nil, errSource + } + data, err := io.ReadAll(io.LimitReader(obj, w.maxBytes+1)) + if err != nil { + return nil, err + } + sum := sha256.Sum256(data) + if int64(len(data)) != src.size || hex.EncodeToString(sum[:]) != src.sha256 { + return nil, errSource + } + return data, nil +} + +func (w *Worker) execute(ctx context.Context, job *claimedJob) error { + ctx, cancel := context.WithTimeout(ctx, w.timeout) + defer cancel() + var src source + err := w.db.QueryRow(ctx, ` + select coalesce(staging_key, ''), content_type, sha256, size_bytes from attachments where id = $1::uuid`, + job.attachmentID).Scan(&src.stagingKey, &src.contentType, &src.sha256, &src.size) + if err != nil { + return err + } + data, err := w.fetch(ctx, src) + if err != nil { + return err + } + variants, err := renderVariants(ctx, data, w.maxPixels) + if err != nil { + return err + } + original := finalKey(job.attachmentID) + if err := w.store.Put(ctx, original, bytes.NewReader(data), int64(len(data)), src.contentType); err != nil { + return err + } + for _, v := range variants { + if err := w.store.Put(ctx, variantKey(job.attachmentID, v.spec), bytes.NewReader(v.data), int64(len(v.data)), v.spec.contentType); err != nil { + return err + } + } + w.inject("after_store") + + tx, err := w.db.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + tag, err := tx.Exec(ctx, ` + update jobs set status = 'ready', lease_owner = null, lease_until = null, last_error = null, updated_at = now() + where id = $1::uuid and status = 'processing' and lease_owner = $2 and attempt = $3`, job.id, job.owner, job.attempt) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + w.log.Warn("lease lost, result discarded", "job_id", job.id, "attempt", job.attempt) + return nil + } + for _, v := range variants { + _, err := tx.Exec(ctx, ` + insert into attachment_variants(attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256) + values($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9) + on conflict (attachment_id, pipeline_version, name) do update + set object_key = excluded.object_key, content_type = excluded.content_type, size_bytes = excluded.size_bytes, + width = excluded.width, height = excluded.height, sha256 = excluded.sha256`, + job.attachmentID, pipelineVersion, v.spec.name, variantKey(job.attachmentID, v.spec), v.spec.contentType, + len(v.data), v.width, v.height, v.sha256) + if err != nil { + return err + } + } + if _, err := tx.Exec(ctx, ` + update attachments set status = 'ready', error_code = null, object_key = $2, updated_at = now() + where id = $1::uuid`, job.attachmentID, original); err != nil { + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + w.log.Info("job ready", "job_id", job.id, "attachment_id", job.attachmentID, "attempt", job.attempt) + w.inject("before_ack") + if err := w.store.Remove(ctx, src.stagingKey); err != nil { + w.log.Warn("staging cleanup deferred to janitor", "attachment_id", job.attachmentID, "err", err.Error()) + } + return nil +} + +func (w *Worker) settle(ctx context.Context, job *claimedJob, permanent bool, code, reason string) error { + tx, err := w.db.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + var maxAttempts int + if err := tx.QueryRow(ctx, `select max_attempts from jobs where id = $1::uuid`, job.id).Scan(&maxAttempts); err != nil { + return err + } + fence := ` where id = $1::uuid and status = 'processing' and lease_owner = $2 and attempt = $3` + if !permanent && job.attempt < maxAttempts { + delay := w.backoff(job.attempt) + tag, err := tx.Exec(ctx, ` + update jobs set status = 'queued', attempt = attempt + 1, next_attempt_at = now() + make_interval(secs => $4), + lease_owner = null, lease_until = null, last_error = $5, updated_at = now()`+fence, + job.id, job.owner, job.attempt, delay.Seconds(), reason) + if err != nil || tag.RowsAffected() == 0 { + return err + } + if _, err := tx.Exec(ctx, ` + insert into outbox(job_id, attempt, available_at, created_at) + values($1::uuid, $2, now() + make_interval(secs => $3), now())`, job.id, job.attempt+1, delay.Seconds()); err != nil { + return err + } + if _, err := tx.Exec(ctx, `update attachments set status = 'queued', updated_at = now() where id = $1::uuid`, job.attachmentID); err != nil { + return err + } + w.log.Warn("job retry scheduled", "job_id", job.id, "attempt", job.attempt, "next_attempt", job.attempt+1, + "delay_ms", delay.Milliseconds(), "reason", reason) + return tx.Commit(ctx) + } + if !permanent { + code = "retry_exhausted" + } + tag, err := tx.Exec(ctx, ` + update jobs set status = 'failed', lease_owner = null, lease_until = null, last_error = $4, updated_at = now()`+fence, + job.id, job.owner, job.attempt, reason) + if err != nil || tag.RowsAffected() == 0 { + return err + } + if _, err := tx.Exec(ctx, ` + update attachments set status = 'failed', error_code = $2, updated_at = now() where id = $1::uuid`, job.attachmentID, code); err != nil { + return err + } + w.log.Error("job failed", "job_id", job.id, "attempt", job.attempt, "error_code", code, "reason", reason) + return tx.Commit(ctx) +} + +func (w *Worker) handle(ctx context.Context, msg redis.XMessage) bool { + var m jobMessage + raw, _ := msg.Values["job"].(string) + if err := json.Unmarshal([]byte(raw), &m); err != nil || m.SchemaVersion != 1 || m.Type != jobType || !isUUID(m.JobID) { + w.log.Warn("invalid job message dropped", "stream_id", msg.ID) + return true + } + job, err := w.claim(ctx, m.JobID) + if err != nil { + w.log.Warn("claim failed", "job_id", m.JobID, "err", err.Error()) + return false + } + if job == nil { + var status string + var attempt int + _ = w.db.QueryRow(ctx, `select status, attempt from jobs where id = $1::uuid`, m.JobID).Scan(&status, &attempt) + w.log.Info("job delivery skipped", "job_id", m.JobID, "message_attempt", m.Attempt, "status", status, "attempt", attempt) + return true + } + err = w.execute(ctx, job) + if err == nil { + return true + } + permanent, code := false, "processing_failed" + switch { + case errors.Is(err, errInvalidImage): + permanent, code = true, "invalid_image" + case errors.Is(err, errSource): + permanent = true + } + if err := w.settle(ctx, job, permanent, code, err.Error()); err != nil { + w.log.Warn("settle failed", "job_id", job.id, "err", err.Error()) + return false + } + return true +} + +func (w *Worker) read(ctx context.Context, consumer, id string, block time.Duration) ([]redis.XMessage, error) { + res, err := w.broker.rdb.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: jobGroup, Consumer: consumer, Streams: []string{jobStream, id}, Count: 1, Block: block, + }).Result() + if len(res) == 0 { + return nil, err + } + return res[0].Messages, err +} + +func (w *Worker) consume(ctx context.Context, consumer string) { + pending := true + nextClaim := time.Now().Add(w.lease) + for ctx.Err() == nil { + var msgs []redis.XMessage + var err error + switch { + case pending: + msgs, err = w.read(ctx, consumer, "0", -1) + pending = len(msgs) > 0 + case time.Now().After(nextClaim): + msgs, _, err = w.broker.rdb.XAutoClaim(ctx, &redis.XAutoClaimArgs{ + Stream: jobStream, Group: jobGroup, Consumer: consumer, MinIdle: w.lease, Start: "0-0", Count: 1, + }).Result() + nextClaim = time.Now().Add(15 * time.Second) + default: + msgs, err = w.read(ctx, consumer, ">", 5*time.Second) + } + if err != nil && !errors.Is(err, redis.Nil) { + if ctx.Err() != nil { + return + } + if strings.Contains(err.Error(), "NOGROUP") { + _ = w.broker.ensureGroup(ctx) + } + w.log.Warn("broker read failed", "consumer", consumer, "err", err.Error()) + time.Sleep(2 * time.Second) + continue + } + for _, msg := range msgs { + if !w.handle(context.WithoutCancel(ctx), msg) { + pending = false + continue + } + if err := w.broker.rdb.XAck(context.WithoutCancel(ctx), jobStream, jobGroup, msg.ID).Err(); err != nil { + w.log.Warn("ack failed", "stream_id", msg.ID, "err", err.Error()) + } + } + } +} + +func (w *Worker) reap(ctx context.Context) error { + rows, err := w.db.Query(ctx, ` + select id::text, attachment_id::text, attempt, coalesce(lease_owner, '') + from jobs where status = 'processing' and lease_until < now() limit 50`) + if err != nil { + return err + } + jobs, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (*claimedJob, error) { + j := &claimedJob{} + return j, row.Scan(&j.id, &j.attachmentID, &j.attempt, &j.owner) + }) + if err != nil { + return err + } + for _, j := range jobs { + w.log.Warn("lease expired", "job_id", j.id, "attempt", j.attempt) + if err := w.settle(ctx, j, false, "", "lease expired"); err != nil { + return err + } + } + return nil +} + +func runWorker(db *pgxpool.Pool, store *Store, broker *Broker, log *slog.Logger, w *Worker, concurrency int) { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + for { + err := broker.ensureGroup(ctx) + if err == nil { + break + } + log.Warn("broker not ready", "err", err.Error()) + select { + case <-ctx.Done(): + return + case <-time.After(2 * time.Second): + } + } + host, _ := os.Hostname() + var wg sync.WaitGroup + wg.Go(func() { + for ctx.Err() == nil { + if err := w.reap(ctx); err != nil && ctx.Err() == nil { + log.Warn("lease reaper failed", "err", err.Error()) + } + select { + case <-ctx.Done(): + case <-time.After(5 * time.Second): + } + } + }) + for i := range concurrency { + consumer := fmt.Sprintf("%s-%d", host, i) + wg.Go(func() { w.consume(ctx, consumer) }) + } + log.Info("worker started", "owner", w.owner, "concurrency", concurrency, "lease_s", w.lease.Seconds()) + <-ctx.Done() + log.Info("worker stopping") + wg.Wait() +} + +func replayJob(ctx context.Context, db *pgxpool.Pool, attachmentID string, maxAttempts int) (string, error) { + tx, err := db.Begin(ctx) + if err != nil { + return "", err + } + defer tx.Rollback(ctx) + var jobID, status string + var attempt int + var lastErr *string + err = tx.QueryRow(ctx, `select id::text, status, attempt, last_error from jobs where attachment_id = $1::uuid for update`, + attachmentID).Scan(&jobID, &status, &attempt, &lastErr) + if errors.Is(err, pgx.ErrNoRows) { + return "not_found", nil + } + if err != nil { + return "", err + } + if status != "failed" { + return "skipped_" + status, nil + } + if _, err := tx.Exec(ctx, `insert into job_replays(job_id, previous_error, replayed_at) values($1::uuid, $2, now())`, jobID, lastErr); err != nil { + return "", err + } + if _, err := tx.Exec(ctx, ` + update jobs set status = 'queued', attempt = attempt + 1, max_attempts = attempt + $2, next_attempt_at = now(), + last_error = null, updated_at = now() + where id = $1::uuid`, jobID, maxAttempts); err != nil { + return "", err + } + if _, err := tx.Exec(ctx, `insert into outbox(job_id, attempt, available_at, created_at) values($1::uuid, $2, now(), now())`, jobID, attempt+1); err != nil { + return "", err + } + if _, err := tx.Exec(ctx, `update attachments set status = 'queued', error_code = null, updated_at = now() where id = $1::uuid`, attachmentID); err != nil { + return "", err + } + return "queued", tx.Commit(ctx) +} + +func listFailed(ctx context.Context, db *pgxpool.Pool) ([]map[string]any, error) { + rows, err := db.Query(ctx, ` + select j.attachment_id::text, j.id::text, j.attempt, coalesce(a.error_code, ''), coalesce(j.last_error, ''), j.updated_at + from jobs j join attachments a on a.id = j.attachment_id + where j.status = 'failed' order by j.updated_at`) + if err != nil { + return nil, err + } + return pgx.CollectRows(rows, func(row pgx.CollectableRow) (map[string]any, error) { + var att, job, code, reason string + var attempt int + var updated time.Time + err := row.Scan(&att, &job, &attempt, &code, &reason, &updated) + return map[string]any{"attachment_id": att, "job_id": job, "attempt": attempt, "error_code": code, + "last_error": reason, "failed_at": formatTime(updated)}, err + }) +}