ЛР 5 - БВТ2304 - Березин - на 4 #21
+5
-14
@@ -1,25 +1,16 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
API_REPLICAS=2
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
CLEANUP_GRACE_SECONDS=600
|
||||
CLEANUP_INTERVAL_SECONDS=300
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_BACKOFF_BASE_MS=1000
|
||||
JOB_BACKOFF_MAX_MS=30000
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
# Изменения lab5 относительно lab4
|
||||
|
||||
## API
|
||||
|
||||
- `POST /chats/{chat_id}/attachments` возвращает `202` со статусом `queued` вместо `201 ready`. Проверки при приёме те же, что в lab4.
|
||||
- Оригинал сначала сохраняется в приватный S3-prefix `staging/<id>`, а не сразу в конечный `attachments/<id>`.
|
||||
- `GET /attachments/{id}` возвращает статусы `queued`, `processing`, `ready`, `failed`, а также `error_code` и `variants`.
|
||||
- `GET /attachments/{id}/content`:
|
||||
- принимает `variant=original|thumbnail|crop|optimized`, неизвестное имя → 400;
|
||||
- для не-ready вложения → 409;
|
||||
- отсутствующий вариант ready-вложения → 404.
|
||||
- `POST /chats/{chat_id}/messages`: прикрепить `failed`-вложение нельзя (409), `queued`/`processing` можно.
|
||||
|
||||
## База данных
|
||||
|
||||
Новая миграция `migrations/0004_jobs.sql`:
|
||||
|
||||
- В `attachments` добавлены `status`, `error_code`, `staging_key`, `updated_at`. `object_key` теперь заполняет worker.
|
||||
- `attachment_variants`: варианты с первичным ключом `(attachment_id, pipeline_version, name)`.
|
||||
- `jobs`: задача обработки (статус, попытка, лимит попыток, время следующей попытки, lease, последняя ошибка).
|
||||
- `outbox`: задания на публикацию в брокер.
|
||||
- `job_replays`: аудит операторских повторов.
|
||||
|
||||
## Новые компоненты
|
||||
|
||||
- `broker`: Valkey Streams, отдельный сервис с volume, `appendfsync always`, `noeviction`, паролем из secrets. Stream `attachments.jobs`, группа `workers`.
|
||||
- `worker`: отдельный процесс (`messenger worker`), 2 потребителя, prefetch 1, лимит 768 MiB.
|
||||
- `broker.go`: формат сообщения по `jobs.schema.json`, публикация, outbox relay (работает в каждой API-реплике).
|
||||
- `worker.go`:
|
||||
- захват задачи с lease и fencing по `lease_owner` и `attempt`;
|
||||
- обработка и фиксация результата;
|
||||
- retry с backoff и jitter;
|
||||
- reaper для задач с истёкшим lease;
|
||||
- повторное чтение своих pending и `XAUTOCLAIM`;
|
||||
- команды `failed` и `replay`.
|
||||
- `imaging.go`:
|
||||
- thumbnail JPEG 256, crop JPEG 128×128, optimized WebP quality 80 (вписан в 1280);
|
||||
- применение EXIF orientation, alpha на белом;
|
||||
- предварительное уменьшение больших изображений для экономии памяти.
|
||||
|
||||
## Изменения существующего кода
|
||||
|
||||
- `attachments.go`: приём в staging, транзакция `attachments` + `jobs` + `outbox`, ответ 202, загрузка вариантов, скачивание по `variant`. Janitor дополнительно чистит `staging/`.
|
||||
- `handlers.go`: проверка failed-вложений при создании сообщения, точка внедрения отказа.
|
||||
- `main.go`: подкоманды `worker`, `failed`, `replay`; запуск relay в API; новые переменные окружения.
|
||||
- `s3.go`: listing по произвольному prefix.
|
||||
- `Dockerfile`: сборка с тегом `nodynamic` (WebP без системных библиотек).
|
||||
- `go.mod`: добавлены `golang.org/x/image`, `github.com/gen2brain/webp`, `github.com/rwcarlsen/goexif`.
|
||||
|
||||
## Инфраструктура и скрипты
|
||||
|
||||
- `compose.yaml`: сервисы `broker` и `worker`, volume `broker_data`, secrets `broker_conf` и `broker_url`, образ `mtusi-messenger:lab5`, переменные для точек внедрения отказа (по умолчанию пусты).
|
||||
- `scripts/mksecrets.sh`: генерирует пароль брокера, `broker.conf` и `broker_url`.
|
||||
- `scripts/restart.sh`: перезапускает также `broker` и `worker`.
|
||||
- `scripts/rolling-update.sh`: обновляет также `worker`.
|
||||
- `.env.example`: `WORKER_CONCURRENCY`, `JOB_MAX_ATTEMPTS`, `JOB_TIMEOUT_SECONDS`, `JOB_BACKOFF_BASE_MS`, `JOB_BACKOFF_MAX_MS`, `STAGING_RETENTION_SECONDS`.
|
||||
|
||||
## Надёжность
|
||||
|
||||
- Работа не теряется после ответа 202: оригинал в S3, задача и outbox в одной транзакции PostgreSQL.
|
||||
- Разрыв COMMIT/publish закрыт transactional outbox.
|
||||
- Повторная доставка и повторы не создают второго набора вариантов: детерминированные S3-ключи и upsert по ключу эффекта.
|
||||
- Зависшие задачи возвращаются в очередь по истечении lease.
|
||||
- Постоянные ошибки (`invalid_image`, `processing_failed`) уходят в failed сразу, временные повторяются до `retry_exhausted`.
|
||||
- Failed-задачи можно повторить через `replay`. Повтор уже готовой задачи ничего не делает.
|
||||
- Повреждение внутри PNG, которое lab4 пропускал (проверялись только заголовок и конец), теперь ловит worker при полном декодировании.
|
||||
|
||||
## Тесты
|
||||
|
||||
Новые:
|
||||
- `tests/stack.py`: общие хелперы.
|
||||
- `tests/variants.py`: размеры, форматы, sha256, без увеличения, alpha, EXIF.
|
||||
- `tests/queue_recovery.py`: сценарий на 3 и повторная доставка.
|
||||
- `tests/faults.py`: 5 сценариев сбоев на 4.
|
||||
- `tests/worker_load.py`: конкуренция, память и timeout worker.
|
||||
|
||||
Адаптированы под 202: `tests/attachments.py`, `tests/s3_checks.py` (плюс проверка очистки staging), `tests/upload_memory.py`.
|
||||
|
||||
Без изменений: `tests/idempotency.py`, `tests/scenario.py`, `tests/image_gen.py`.
|
||||
|
||||
## Отчёт и доказательства
|
||||
|
||||
- `REPORT.md` переписан под lab5.
|
||||
- Новые файлы в `evidence/`: `variants.txt`, `queue-recovery.txt`, `faults.txt`, `worker-load.json`.
|
||||
- Остальные файлы evidence получены заново на стенде lab5.
|
||||
|
||||
## Исправления после ревью
|
||||
|
||||
- Авторизация проверяет сессию только в PostgreSQL одним запросом (`sessions` ⋈ `users`, `expires_at > now()`); запись кэша не принимается за доказательство действительности токена. Код кэша сессий (`cache.go`) удалён.
|
||||
- Добавлен `tests/session_revocation.py`: Valkey остановлен → logout 204 → Valkey запущен → прежний токен 401.
|
||||
- Тесты `image_gen.py`, `idempotency.py`, `s3_checks.py`, `scenario.py`, `upload_memory.py` были не добавлены в git, из-за чего `attachments.py` и `worker_load.py` падали с `ModuleNotFoundError: image_gen`.
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
FROM golang:1.26-alpine@sha256:c95332c2af86b6d89b91bd0500f4b9529ccbd090a0d1855c6d1ceaa142ae8615 AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY *.go ./
|
||||
COPY migrations ./migrations
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -tags nodynamic -trimpath -ldflags="-s -w" -o /out/messenger .
|
||||
|
||||
FROM alpine:3.21@sha256:ce64758a109eb420d874a118f87920e625e12d3634e03b4a5573fd9f6e5d3507
|
||||
RUN adduser -D -u 10001 app
|
||||
WORKDIR /app
|
||||
COPY --from=build /out/messenger /app/messenger
|
||||
USER 10001
|
||||
EXPOSE 8080
|
||||
ENV APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab
|
||||
STOPSIGNAL SIGTERM
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD wget -qO- http://127.0.0.1:8080/health/ready >/dev/null || exit 1
|
||||
ENTRYPOINT ["/app/messenger"]
|
||||
@@ -1,37 +1,177 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Березин Герман / БВТ2304
|
||||
- Целевая оценка: 4
|
||||
- Commit/PR: после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: Go 1.26, pgx, go-redis (клиент Valkey Streams), minio-go v7, golang.org/x/image/draw (ресемплинг), github.com/gen2brain/webp (libwebp, транспилированный в Go), github.com/rwcarlsen/goexif (EXIF orientation); PostgreSQL 17, Valkey 8.1, SeaweedFS, HAProxy 3.0; документация RabbitMQ reliability и Redis Streams из README; код lab4 перенесён за основу; ИИ-ассистент использован при разработке
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Docker 29 с Compose v2 и Python 3.9+. Для `tests/variants.py` нужен Pillow (проверяет реальные пиксели JPEG/WebP).
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
sh scripts/mksecrets.sh
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
make check && make test
|
||||
```
|
||||
|
||||
Порядок старта: postgres, valkey, broker, s3 → `migrate` → `s3-init` → `api` × 2, `worker`, `janitor` → `lb`. Публичный адрес: `http://localhost:8080`. После первого старта S3 запись становится доступна через 10–30 s.
|
||||
|
||||
Остановка без потери данных: `docker compose --env-file .env stop`. Намеренное удаление учебных данных: `docker compose --env-file .env down -v`.
|
||||
|
||||
Операторские команды:
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env stop worker
|
||||
docker compose --env-file .env up -d --no-deps worker
|
||||
docker compose --env-file .env run --rm --no-deps worker failed
|
||||
docker compose --env-file .env run --rm --no-deps worker replay <attachment_id>
|
||||
```
|
||||
|
||||
Проверки (стенд запущен):
|
||||
|
||||
```sh
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
python3 tests/variants.py
|
||||
python3 tests/queue_recovery.py
|
||||
python3 tests/faults.py
|
||||
python3 tests/worker_load.py --output evidence/worker-load.json
|
||||
python3 tests/attachments.py
|
||||
python3 tests/s3_checks.py
|
||||
python3 tests/upload_memory.py --output evidence/upload-memory.json
|
||||
python3 tests/idempotency.py
|
||||
python3 tests/session_revocation.py
|
||||
python3 tests/scenario.py --mode failover-each --output evidence/failover-each.json
|
||||
python3 tests/scenario.py --mode rolling --output evidence/rolling.json
|
||||
```
|
||||
|
||||
После `make test-failover` реплику возвращают командой `docker compose --env-file .env up -d --wait api`. `faults.py`, `queue_recovery.py`, `worker_load.py` сами останавливают и пересоздают worker/api/broker/s3 и возвращают их в исходное состояние.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
клиент → HAProxy → api ×2 ──(1) S3 staging/<id>
|
||||
└──(2) PostgreSQL: attachments(queued) + jobs + outbox — одна транзакция → 202
|
||||
api: outbox relay ──XADD──→ broker (Valkey Stream attachments.jobs, group workers)
|
||||
worker ──XREADGROUP──→ claim (lease) → S3 attachments/<id>/... → COMMIT ready → XACK
|
||||
```
|
||||
|
||||
| Роль | Сервис | Образ | Готовность | Volume | Сеть | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| api ×2 | `api` | собирается из исходников (`mtusi-messenger:lab5`) | `/health/ready` (PostgreSQL) | нет | backend, storage | secrets: database_url, cache_url, broker_url, S3-ключи API |
|
||||
| worker | `worker` | тот же образ, команда `worker` | нет HTTP, healthcheck отключён | нет | backend, storage | database_url, broker_url, S3-ключи API |
|
||||
| postgres | `postgres` | postgres:17-alpine@sha256:b0f9560a… | `pg_isready` | pg_data | backend | postgres_password |
|
||||
| broker | `broker` | valkey/valkey:8.1-alpine@sha256:081c2f5c… | `valkey-cli ping` с паролем | broker_data | backend | broker.conf (`requirepass`) |
|
||||
| s3 | `s3` | chrislusf/seaweedfs@sha256:c2658820… | `/dir/assign` выдаёт fid | s3_data | storage | s3_config.json |
|
||||
| load_balancer | `lb` | haproxy:3.0-alpine@sha256:cf3af0fa… | проверки `/health/ready` бэкендов | нет | backend, 127.0.0.1:8080 | нет |
|
||||
| Valkey (сессии проверяются только в PostgreSQL) | `valkey` | valkey/valkey:8.1-alpine | `valkey-cli ping` | valkey_data | backend | нет |
|
||||
|
||||
Bootstrap: `migrate` применяет `migrations/*.sql` под advisory lock, повторный запуск пропускает применённые. `s3-init` создаёт bucket, если его нет. Consumer group `workers` создают API (перед первой публикацией) и worker при старте (`XGROUP CREATE … 0 MKSTREAM`, ошибка BUSYGROUP игнорируется).
|
||||
|
||||
- **Брокер.** Valkey Streams в отдельном сервисе `broker` с `appendonly yes`, `appendfsync always`, `noeviction`. Доставка at least once: сообщение остаётся в pending (PEL) до `XACK`. Publisher confirm: ответ на `XADD` приходит после записи в AOF с fsync. Consumer ack: `XACK` после COMMIT результата в PostgreSQL. В сообщении только поля `jobs.schema.json` (поле `job` — JSON с `schema_version`, `job_id`, `type`, `attachment_id`, `pipeline_version`, `attempt`, `created_at`), без bytes, URL и credentials.
|
||||
- **Что сохранено к 202.** Полный проверенный оригинал лежит в приватном S3-prefix `staging/<attachment_id>` (тот же bucket, volume s3_data). Одной транзакцией сохранены строки `attachments` (status `queued`, staging_key, sha256, размер), `jobs` (attempt 1) и `outbox`. Проверки на приёме те же, что в lab4: 413/415/400/404, хвост и заголовок изображения, не более 40 млн пикселей.
|
||||
- **Transactional outbox.** В каждой API-реплике работает relay: `select … from outbox where published_at is null and available_at <= now() for update skip locked`, затем `XADD` и `published_at = now()` в той же транзакции. Падение между COMMIT и publish оставляет неопубликованную строку, её публикует любая живая реплика. Падение после `XADD` до COMMIT пометки даёт повторную публикацию: это дубль, worker обработает его как no-op.
|
||||
- **Worker.** Отдельный процесс (`messenger worker`), `WORKER_CONCURRENCY=2` потребителя, у каждого `COUNT 1`: prefetch 1, не больше двух задач одновременно. При старте потребитель дочитывает свои pending-сообщения (`XREADGROUP … 0`), каждые 15 s забирает зависшие чужие (`XAUTOCLAIM` с min-idle = lease).
|
||||
- **Ключ идемпотентности.** Логическая работа — `jobs.id` (job_id), он один на вложение и не меняется при redelivery. Ключ эффекта — `(attachment_id, pipeline_version, name)`: первичный ключ `attachment_variants`, вставка через `on conflict do update`. S3-ключи детерминированы: `attachments/<id>/original`, `attachments/<id>/v1/{thumbnail.jpg,crop.jpg,optimized.webp}`. Повтор перезаписывает те же объекты и не создаёт второго набора.
|
||||
- **Захват и lease.** `update jobs set status='processing', lease_owner, lease_until = now() + lease where status='queued' and next_attempt_at <= now()`. Сообщение для задачи в любом другом состоянии подтверждается без работы (`job delivery skipped`), поэтому повторная доставка ready-задачи — no-op. Lease = `JOB_TIMEOUT_SECONDS` + 15 s. Фиксация результата и перевод в retry/failed выполняются только при совпадении `lease_owner` и `attempt`. Worker, потерявший lease, не перезапишет состояние (`lease lost, result discarded`).
|
||||
- **Зависший или долгий.** Задача ограничена контекстом `JOB_TIMEOUT_SECONDS` (60 s), поэтому живой worker не может законно работать дольше lease. Каждые 5 s reaper в worker переводит задачи с истёкшим lease в `queued` со следующей попыткой или в `failed`. Новый захват получает новый `attempt`. Старый процесс не сможет зафиксировать результат из-за fencing. Запись тех же детерминированных объектов безвредна.
|
||||
- **Retry.** Временные ошибки (S3, PostgreSQL, timeout, истёкший lease) дают `queued` с `attempt+1`, новой строкой outbox с `available_at = now() + delay` и записью причины в `jobs.last_error`. Задержка `d = min(JOB_BACKOFF_BASE_MS·2^(n−1), JOB_BACKOFF_MAX_MS)` выбирается случайно в `[d/2, d]` (equal jitter). Всего `JOB_MAX_ATTEMPTS = 5` попыток, затем `failed` с `retry_exhausted`. Постоянные ошибки сразу дают `failed` без повторов: `invalid_image` (файл не декодируется, превышен лимит пикселей), `processing_failed` (staging отсутствует или не совпадает sha256/размер).
|
||||
- **Реестр failed и replay.** Реестр — `jobs` со статусом `failed` (`worker failed` выводит attachment_id, attempt, error_code, last_error). `worker replay <attachment_id>` переводит в `queued` только задачу в `failed`: даёт новый бюджет из `JOB_MAX_ATTEMPTS` попыток, пишет аудит в `job_replays` и строку outbox. Для `ready`/`queued`/`processing` результат `skipped_<status>`, дубля нет.
|
||||
- **Варианты.** Все варианты строятся из пикселей и кодируются заново, поэтому EXIF и прочие метаданные не копируются. EXIF orientation (JPEG) применяется, alpha компонуется на белый, ресемплинг Catmull-Rom. Изображения больше 2560 px сначала уменьшаются вдвое блоком 2×2, чтобы ограничить промежуточный буфер. Итоговые размеры считаются от исходных.
|
||||
- `thumbnail`: JPEG q=85, вписан в 256×256, без увеличения, размеры округляются вниз, минимум 1 px; fixture 320×200 → 256×160.
|
||||
- `crop`: JPEG q=85, центральный квадрат, 128×128, маленькие исходники увеличиваются.
|
||||
- `optimized`: WebP lossy, quality 80, method 4, вписан в 1280×1280 без увеличения. Уменьшение каждого маленького файла не обещается.
|
||||
- **Состояния API.** POST upload → 202 `queued`. GET метаданных возвращает `queued` / `processing` / `ready` / `failed`. Для `ready` в `variants` три записи с размерами и sha256 реального файла. `/content` для не-ready → 409. `variant=` с неизвестным именем → 400. Отсутствующий вариант ready-объекта → 404. Прикрепить `failed`-вложение к сообщению → 409, `queued`/`processing` можно. Ready ставится одной транзакцией после записи оригинала и всех трёх вариантов.
|
||||
- **Staging.** Удаляется worker'ом после COMMIT ready. Если удаление не прошло, `janitor` удаляет staging ready-вложений и staging без строки (старше `CLEANUP_GRACE_SECONDS`). Staging `failed` хранится `STAGING_RETENTION_SECONDS` (24 ч) ради replay. Staging `queued`/`processing` не удаляется.
|
||||
- **Критичные зависимости.** `/health/ready` проверяет только PostgreSQL. Брокер не нужен синхронному пути: при его недоступности загрузки принимаются, задания копятся в outbox. Upload и download требуют S3 (иначе 503).
|
||||
- **Внедрение отказов.** `FAULT_POINT` по умолчанию пуст. В тестах через `API_FAULT_POINT=after_commit` (выход процесса после COMMIT, до ответа и publish) и `WORKER_FAULT_POINT=after_store|before_ack` (выход после записи в S3 до COMMIT / после COMMIT до ack).
|
||||
- Перенесено из lab4 без изменения смысла: сессии, идемпотентность сообщений, HAProxy, приватный bucket и отдельные ключи, janitor (дополнен очисткой staging), лимиты загрузки. Изменено: upload 201 → 202, `attachments.object_key` заполняется worker'ом, добавлена миграция `0004_jobs.sql`.
|
||||
|
||||
### Окна отказа
|
||||
|
||||
| Окно | Что происходит | Восстановление |
|
||||
| --- | --- | --- |
|
||||
| после staging до записи задания | объект в `staging/` без строки, клиент получает 503 или обрыв | janitor удаляет его после grace period |
|
||||
| после COMMIT до publish | строка outbox не опубликована | relay любой API-реплики публикует; проверено (`faults.py`, 1) |
|
||||
| после publish до confirm/пометки | сообщение в stream, строка outbox не помечена | повторная публикация → дубль → no-op при захвате |
|
||||
| после записи S3 до COMMIT результата | задача `processing`, объекты записаны | lease истекает → reaper → попытка 2 перезаписывает те же ключи; проверено (`faults.py`, 2) |
|
||||
| после COMMIT результата до ack | ready, сообщение в PEL | после рестарта worker дочитывает pending → no-op → ack; проверено (`faults.py`, 3) |
|
||||
|
||||
### Соответствие .env.example
|
||||
|
||||
| Переменная | Где используется |
|
||||
| --- | --- |
|
||||
| `API_REPLICAS` | `deploy.replicas` сервиса api |
|
||||
| `S3_REGION`, `S3_BUCKET` | api, worker, janitor, s3-init, генерация прав в `mksecrets.sh` |
|
||||
| `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` | api (приём) и worker (повторная проверка) |
|
||||
| `WORKER_CONCURRENCY` | число потребителей worker |
|
||||
| `JOB_MAX_ATTEMPTS` | api записывает в `jobs.max_attempts`, replay выдаёт новый бюджет |
|
||||
| `JOB_TIMEOUT_SECONDS` | таймаут задачи, lease = значение + 15 s |
|
||||
| `JOB_BACKOFF_BASE_MS`, `JOB_BACKOFF_MAX_MS` | backoff повторов |
|
||||
| `STAGING_RETENTION_SECONDS` | хранение staging failed-вложений (janitor) |
|
||||
| `CLEANUP_GRACE_SECONDS`, `CLEANUP_INTERVAL_SECONDS` | janitor |
|
||||
| `BROKER_URL_FILE`, `DATABASE_URL_FILE`, `S3_*_FILE` | задаются в compose как пути к secrets |
|
||||
|
||||
### Лимиты worker
|
||||
|
||||
| Ограничение | Значение |
|
||||
| --- | --- |
|
||||
| Bytes | размер staging должен совпасть с метаданными; чтение не более `MAX_UPLOAD_BYTES`, sha256 сверяется |
|
||||
| Пиксели | `image.DecodeConfig` до декодирования, не более `MAX_IMAGE_PIXELS` (40 млн), иначе `invalid_image` |
|
||||
| Время | `JOB_TIMEOUT_SECONDS` на задачу (S3, БД, проверки между этапами) |
|
||||
| Память | 2 задачи одновременно, `GOMEMLIMIT=600MiB`, лимит контейнера 768 MiB, CPU 1.0 |
|
||||
| Конкуренция | `WORKER_CONCURRENCY=2`, prefetch (`COUNT`) 1 на потребителя |
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Публичный smoke (202 → ready, оригинал, настоящий JPEG 256×160) | `make check && make test` | PASS | evidence/smoke.txt |
|
||||
| Отозванная сессия не воскресает после перезапуска Valkey | `tests/session_revocation.py` | Valkey остановлен → logout 204 → Valkey запущен → прежний токен 401 | evidence/session-revocation.txt |
|
||||
| Минимум lab4 сохранён (валидация, доступ, приватный bucket, отказ S3, идемпотентность) | `tests/attachments.py`, `tests/s3_checks.py`, `tests/idempotency.py` | PASS: 413/415/400/404, анонимный доступ 403, 503 при остановленном S3 | evidence/attachments.txt, evidence/s3-checks.txt, evidence/idempotency.txt |
|
||||
| Restart всех stores с volumes | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; перезапущены postgres, valkey, broker, s3, api, lb, janitor, worker (состояния до/после в файле) | evidence/persistence.txt |
|
||||
| Failover каждой API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS для api-1 и api-2, 0–1 неуспешная попытка, восстановление 3.5 s | evidence/failover-1.txt, evidence/failover-2.txt |
|
||||
| Worker остановлен → upload 202 → сообщение → content 409 → рестарт broker и API → worker запущен → ready | `tests/queue_recovery.py` | задача есть в stream до и после рестарта broker, статус `queued` без worker, после старта ready, attempt 1, 3 варианта | evidence/queue-recovery.txt |
|
||||
| Повторная доставка не размножает варианты | `tests/queue_recovery.py` (ручной `XADD` дубля) | `job delivery skipped`, ack, `updated_at` не изменился, 3 варианта | evidence/queue-recovery.txt |
|
||||
| Thumbnail, crop 128×128, optimized WebP, качество/метаданные | `tests/variants.py` | размеры и форматы по реальным байтам, sha256 совпадает, без увеличения, alpha → белый, EXIF orientation 6 применена, EXIF в вариантах нет | evidence/variants.txt |
|
||||
| Crash между COMMIT и publish | `tests/faults.py`, сценарий 1 | API упал после COMMIT (202 не получен), outbox опубликован другой репликой, ready attempt 1 | evidence/faults.txt |
|
||||
| Остановка worker после записи объекта до COMMIT, timeout/lease | `tests/faults.py`, сценарий 2 | lease истёк, reaper вернул в очередь, ready attempt 2, вариантов 3 | evidence/faults.txt |
|
||||
| Остановка worker после COMMIT до ack | `tests/faults.py`, сценарий 3 | после рестарта pending доставлен повторно, no-op, attempt 1, PEL пуст | evidence/faults.txt |
|
||||
| Backoff с jitter, ограниченный retry, failed, replay | `tests/faults.py`, сценарий 4 | S3 остановлен: 4 повтора с задержками 931/1284/2760/7484 мс (окна [d/2, d]), `retry_exhausted`, задача в реестре, content и прикрепление → 409; replay → ready, повторный replay → `skipped_ready`, аудит 1 строка | evidence/faults.txt |
|
||||
| Постоянная ошибка без повторов | `tests/faults.py`, сценарий 5 | PNG с повреждённым IDAT (проходит проверку API) → `invalid_image` на попытке 1 | evidence/faults.txt |
|
||||
| Ограничение конкуренции, памяти и времени worker | `tests/worker_load.py` | 6 изображений по 39.9 млн пикселей + 20 малых: одновременно не более 2 processing, пик RSS 617 MiB при лимите 768 MiB, без OOM; при `JOB_TIMEOUT_SECONDS=1` — `context deadline exceeded`, 5 попыток, `retry_exhausted` | evidence/worker-load.json |
|
||||
| Очистка staging после безопасной фиксации | `tests/s3_checks.py` | staging ready-вложения удалён, staging queued-задачи сохранён janitor'ом, staging без строки удалён | evidence/s3-checks.txt |
|
||||
| Память API при загрузках | `tests/upload_memory.py` | 12 потоков × 3 файла по 9.3 MiB, 36/36 → 202, пик api 119 / 100 MiB при лимите 192 | evidence/upload-memory.json |
|
||||
| Отказ реплики и rolling update под нагрузкой | `tests/scenario.py` | см. эксперименты | evidence/failover-each.json, evidence/rolling.json |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Стенд: один хост, Docker 29.9.2, Compose 5.6.1, лимиты контейнеров как выше. Нагрузка сообщений — 8 потоков, closed-loop, повтор с тем же ключом. Нагрузка worker — 6 PNG 6320×6320 (116 545 байт, однотонные: проверка лимита пикселей, а не сети) и 20 копий fixture.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Worker остановлен, upload | 202, работа ждёт | `queued`, content 409, текст работает | после старта worker ready, потерь нет |
|
||||
| Restart broker и API с volumes при остановленном worker | задача не теряется | job_id в stream до и после | потерь нет |
|
||||
| Падение API после COMMIT | задача не теряется без ручного вмешательства | outbox опубликован живой репликой | ready в пределах секунд, потерь нет |
|
||||
| Падение worker после записи S3 | повтор того же результата | ready с attempt 2 после истечения lease 25 s (тест с `JOB_TIMEOUT_SECONDS=10`) | ≈ lease + 5 s, дублей нет |
|
||||
| Падение worker после COMMIT | повторная доставка = no-op | skipped, ack | сразу после рестарта |
|
||||
| S3 недоступен для worker | ограниченные повторы, failed | 5 попыток за ≈ 13 s, `retry_exhausted`, replay → ready | после replay, потерь нет |
|
||||
| 6×40 Мп + 20 малых | не более 2 задач одновременно, память в лимите | все ready за 13.1 s, пик RSS 617 MiB / 768 | без OOM и рестартов |
|
||||
| Остановка каждой API-реплики под нагрузкой | запись продолжается | 2 и 0 неуспешных попыток, RPS 439 / 447 против 524 до отказа | возврат реплики около 6.5 s |
|
||||
| Rolling update | без простоя | 4 неуспешные попытки, RPS 615 | старые реплики не отвечают |
|
||||
|
||||
Первая версия без предварительного уменьшения упиралась в лимит 512 MiB (RSS 780 MiB при двух 40-Мп задачах). Причина — промежуточный буфер Catmull-Rom размером ширина_цели × высота_источника × 32 байта. После уменьшения блоком 2×2 пик стал около 600 MiB, а время двух таких задач сократилось с 11 до 4.6 s.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Декодирование и ресемплинг одного этапа нельзя прервать: timeout проверяется между этапами и на S3/БД-операциях. Длительность этапа ограничена лимитом пикселей (около 2 s на 40 Мп).
|
||||
- Broker, PostgreSQL, S3 и HAProxy — единичные экземпляры. `appendfsync always` надёжнее, но медленнее everysec. Stream не обрезается (`XTRIM` не реализован), он растёт на одну запись на попытку.
|
||||
- Relay работает только в API: если остановлены обе реплики API, принятые задания ждут в outbox. Задержка relay до 300 ms.
|
||||
- Имена потребителей привязаны к hostname контейнера: после пересоздания worker старые pending переходят к новым через `XAUTOCLAIM` только после lease (75 s), а старые consumer'ы остаются в группе.
|
||||
- EXIF orientation читается только у JPEG; eXIf-чанк PNG не учитывается.
|
||||
- `pipeline_version` фиксирован (1). Версионирование нескольких pipeline (трек 5) не делалось.
|
||||
- Replay — операторская CLI-команда без HTTP API. `last_error` хранит внутренний текст ошибки только в БД и наружу не выдаётся.
|
||||
- WebP кодируется транспилированным libwebp (без CGo): медленнее нативного, но без системных библиотек.
|
||||
- Секреты лежат в `./secrets` вне Git. Для чтения процессом с UID 10001 у них права 644: это допустимо только для локального стенда.
|
||||
|
||||
+530
@@ -0,0 +1,530 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"hash"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
const (
|
||||
objectPrefix = "attachments/"
|
||||
stagingPrefix = "staging/"
|
||||
probeHead = 256 << 10
|
||||
tailLen = 12
|
||||
)
|
||||
|
||||
var (
|
||||
pngMagic = []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}
|
||||
pngTail = []byte{0, 0, 0, 0, 'I', 'E', 'N', 'D', 0xae, 0x42, 0x60, 0x82}
|
||||
jpegMagic = []byte{0xff, 0xd8, 0xff}
|
||||
jpegTail = []byte{0xff, 0xd9}
|
||||
)
|
||||
|
||||
type limits struct {
|
||||
maxBytes int64
|
||||
maxPixels int64
|
||||
uploadSlots int
|
||||
maxAttempts int
|
||||
}
|
||||
|
||||
type attachment struct {
|
||||
ID, ChatID, OwnerID, Filename, ContentType, SHA256, Key, Status string
|
||||
ErrorCode *string
|
||||
Size int64
|
||||
Created, Updated time.Time
|
||||
Variants []variant
|
||||
}
|
||||
|
||||
type variant struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
Size int64 `json:"size_bytes"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
SHA256 string `json:"sha256"`
|
||||
key string
|
||||
}
|
||||
|
||||
func (x *attachment) json() map[string]any {
|
||||
variants := x.Variants
|
||||
if variants == nil {
|
||||
variants = []variant{}
|
||||
}
|
||||
return map[string]any{
|
||||
"id": x.ID,
|
||||
"chat_id": x.ChatID,
|
||||
"owner_id": x.OwnerID,
|
||||
"filename": x.Filename,
|
||||
"content_type": x.ContentType,
|
||||
"size_bytes": x.Size,
|
||||
"sha256": x.SHA256,
|
||||
"status": x.Status,
|
||||
"variants": variants,
|
||||
"error_code": x.ErrorCode,
|
||||
"created_at": formatTime(x.Created),
|
||||
"updated_at": formatTime(x.Updated),
|
||||
}
|
||||
}
|
||||
|
||||
type probe struct {
|
||||
sum hash.Hash
|
||||
size int64
|
||||
head []byte
|
||||
tail []byte
|
||||
}
|
||||
|
||||
func newProbe() *probe {
|
||||
return &probe{sum: sha256.New()}
|
||||
}
|
||||
|
||||
func (p *probe) Write(b []byte) (int, error) {
|
||||
p.sum.Write(b)
|
||||
p.size += int64(len(b))
|
||||
if room := probeHead - len(p.head); room > 0 {
|
||||
if room > len(b) {
|
||||
room = len(b)
|
||||
}
|
||||
p.head = append(p.head, b[:room]...)
|
||||
}
|
||||
p.tail = append(p.tail, b...)
|
||||
if len(p.tail) > tailLen {
|
||||
p.tail = append(p.tail[:0], p.tail[len(p.tail)-tailLen:]...)
|
||||
}
|
||||
return len(b), nil
|
||||
}
|
||||
|
||||
func (p *probe) digest() string {
|
||||
return hex.EncodeToString(p.sum.Sum(nil))
|
||||
}
|
||||
|
||||
var (
|
||||
errNotImage = errors.New("not an image")
|
||||
errTooBig = errors.New("too large")
|
||||
)
|
||||
|
||||
func (p *probe) verify(contentType string, maxPixels int64) error {
|
||||
cfg, format, err := image.DecodeConfig(bytes.NewReader(p.head))
|
||||
if err != nil {
|
||||
return errNotImage
|
||||
}
|
||||
var tail []byte
|
||||
switch contentType {
|
||||
case "image/png":
|
||||
if format != "png" {
|
||||
return errNotImage
|
||||
}
|
||||
tail = pngTail
|
||||
case "image/jpeg":
|
||||
if format != "jpeg" {
|
||||
return errNotImage
|
||||
}
|
||||
tail = jpegTail
|
||||
}
|
||||
if !bytes.HasSuffix(p.tail, tail) {
|
||||
return errNotImage
|
||||
}
|
||||
if int64(cfg.Width)*int64(cfg.Height) > maxPixels {
|
||||
return errTooBig
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func magicMatches(contentType string, head []byte) bool {
|
||||
switch contentType {
|
||||
case "image/png":
|
||||
return bytes.HasPrefix(head, pngMagic)
|
||||
case "image/jpeg":
|
||||
return bytes.HasPrefix(head, jpegMagic)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func validFilename(name string) bool {
|
||||
n := utf8.RuneCountInString(name)
|
||||
if n < 1 || n > 255 || !utf8.ValidString(name) {
|
||||
return false
|
||||
}
|
||||
for _, r := range name {
|
||||
if r == '/' || r == '\\' || r == 0 || unicode.IsControl(r) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type trackedReader struct {
|
||||
r io.Reader
|
||||
err error
|
||||
}
|
||||
|
||||
func (t *trackedReader) Read(b []byte) (int, error) {
|
||||
n, err := t.r.Read(b)
|
||||
if err != nil && err != io.EOF && t.err == nil {
|
||||
t.err = err
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func isTooLarge(err error) bool {
|
||||
var mbe *http.MaxBytesError
|
||||
return errors.As(err, &mbe)
|
||||
}
|
||||
|
||||
func (a *App) dropObject(key string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := a.store.Remove(ctx, key); err != nil {
|
||||
a.log.Warn("object cleanup failed", "key", key, "err", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) handleUploadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
reqID := reqIDFrom(r)
|
||||
u, _ := a.authUser(w, r)
|
||||
if u == nil {
|
||||
return
|
||||
}
|
||||
chatID := r.PathValue("chat_id")
|
||||
if !isUUID(chatID) {
|
||||
a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id")
|
||||
return
|
||||
}
|
||||
ok, err := a.isMember(r.Context(), chatID, u.ID)
|
||||
if err != nil {
|
||||
a.writeErr(w, reqID, 503, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
a.writeErr(w, reqID, 404, "not_found", "chat not found")
|
||||
return
|
||||
}
|
||||
select {
|
||||
case a.uploads <- struct{}{}:
|
||||
defer func() { <-a.uploads }()
|
||||
default:
|
||||
w.Header().Set("Retry-After", "1")
|
||||
a.writeErr(w, reqID, 429, "rate_limited", "too many concurrent uploads")
|
||||
return
|
||||
}
|
||||
filename := r.URL.Query().Get("filename")
|
||||
if !validFilename(filename) {
|
||||
a.writeErr(w, reqID, 400, "invalid_request", "invalid filename")
|
||||
return
|
||||
}
|
||||
contentType := strings.ToLower(strings.TrimSpace(strings.Split(r.Header.Get("Content-Type"), ";")[0]))
|
||||
if contentType != "image/png" && contentType != "image/jpeg" {
|
||||
a.writeErr(w, reqID, 415, "unsupported_media_type", "Content-Type must be image/png or image/jpeg")
|
||||
return
|
||||
}
|
||||
if r.ContentLength > a.limits.maxBytes {
|
||||
a.writeErr(w, reqID, 413, "payload_too_large", "file too large")
|
||||
return
|
||||
}
|
||||
body := &trackedReader{r: http.MaxBytesReader(w, r.Body, a.limits.maxBytes)}
|
||||
head := make([]byte, 16)
|
||||
n, _ := io.ReadFull(body, head)
|
||||
if body.err != nil {
|
||||
if isTooLarge(body.err) {
|
||||
a.writeErr(w, reqID, 413, "payload_too_large", "file too large")
|
||||
} else {
|
||||
a.writeErr(w, reqID, 400, "invalid_request", "upload interrupted")
|
||||
}
|
||||
return
|
||||
}
|
||||
head = head[:n]
|
||||
if !magicMatches(contentType, head) {
|
||||
a.writeErr(w, reqID, 415, "unsupported_media_type", "content does not match image type")
|
||||
return
|
||||
}
|
||||
|
||||
id := newUUID()
|
||||
key := stagingPrefix + id
|
||||
pr := newProbe()
|
||||
size := r.ContentLength
|
||||
if size < 0 {
|
||||
size = -1
|
||||
}
|
||||
src := io.TeeReader(io.MultiReader(bytes.NewReader(head), body), pr)
|
||||
putCtx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
|
||||
defer cancel()
|
||||
if err := a.store.Put(putCtx, key, src, size, contentType); err != nil {
|
||||
a.dropObject(key)
|
||||
switch {
|
||||
case isTooLarge(body.err):
|
||||
a.writeErr(w, reqID, 413, "payload_too_large", "file too large")
|
||||
case body.err != nil:
|
||||
a.writeErr(w, reqID, 400, "invalid_request", "upload interrupted")
|
||||
default:
|
||||
a.log.Error("s3 put failed", "request_id", reqID, "err", err.Error())
|
||||
a.writeErr(w, reqID, 503, "unavailable", "storage unavailable")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := pr.verify(contentType, a.limits.maxPixels); err != nil {
|
||||
a.dropObject(key)
|
||||
if err == errTooBig {
|
||||
a.writeErr(w, reqID, 413, "payload_too_large", "image too large")
|
||||
} else {
|
||||
a.writeErr(w, reqID, 415, "unsupported_media_type", "invalid image")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
now := nowUTC()
|
||||
att := &attachment{
|
||||
ID: id, ChatID: chatID, OwnerID: u.ID, Filename: filename, ContentType: contentType,
|
||||
SHA256: pr.digest(), Size: pr.size, Status: "queued", Created: now, Updated: now,
|
||||
}
|
||||
if err := a.enqueue(r.Context(), att, key); err != nil {
|
||||
a.dropObject(key)
|
||||
a.log.Error("enqueue failed", "request_id", reqID, "err", err.Error())
|
||||
a.writeErr(w, reqID, 503, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
if a.fault == "after_commit" {
|
||||
a.log.Error("fault injected", "point", "after_commit", "attachment_id", att.ID)
|
||||
os.Exit(3)
|
||||
}
|
||||
a.writeJSON(w, 202, att.json())
|
||||
}
|
||||
|
||||
func (a *App) enqueue(ctx context.Context, att *attachment, key string) error {
|
||||
ctx, cancel := dbCtx(ctx)
|
||||
defer cancel()
|
||||
tx, err := a.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
if _, err := tx.Exec(ctx, `
|
||||
insert into attachments(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key, status, created_at, updated_at)
|
||||
values($1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, 'queued', $9, $9)`,
|
||||
att.ID, att.ChatID, att.OwnerID, att.Filename, att.ContentType, att.Size, att.SHA256, key, att.Created); err != nil {
|
||||
return err
|
||||
}
|
||||
jobID := newUUID()
|
||||
if _, err := tx.Exec(ctx, `
|
||||
insert into jobs(id, attachment_id, pipeline_version, status, attempt, max_attempts, next_attempt_at, created_at, updated_at)
|
||||
values($1::uuid, $2::uuid, $3, 'queued', 1, $4, $5, $5, $5)`,
|
||||
jobID, att.ID, pipelineVersion, a.limits.maxAttempts, att.Created); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
insert into outbox(job_id, attempt, available_at, created_at) values($1::uuid, 1, $2, $2)`, jobID, att.Created); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (a *App) loadAttachment(ctx context.Context, id, userID string) (*attachment, error) {
|
||||
var x attachment
|
||||
err := a.db.QueryRow(ctx, `
|
||||
select a.id::text, a.chat_id::text, a.owner_id::text, a.filename, a.content_type,
|
||||
a.size_bytes, a.sha256, coalesce(a.object_key, ''), a.status, a.error_code, a.created_at, a.updated_at
|
||||
from attachments a
|
||||
join chat_members m on m.chat_id = a.chat_id and m.user_id = $2::uuid
|
||||
where a.id = $1::uuid`, id, userID).
|
||||
Scan(&x.ID, &x.ChatID, &x.OwnerID, &x.Filename, &x.ContentType, &x.Size, &x.SHA256, &x.Key, &x.Status, &x.ErrorCode, &x.Created, &x.Updated)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if x.Status != "ready" {
|
||||
return &x, nil
|
||||
}
|
||||
rows, err := a.db.Query(ctx, `
|
||||
select name, content_type, size_bytes, width, height, sha256, object_key
|
||||
from attachment_variants where attachment_id = $1::uuid and pipeline_version = $2
|
||||
order by array_position(array['thumbnail', 'crop', 'optimized'], name)`, id, pipelineVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
x.Variants, err = pgx.CollectRows(rows, func(row pgx.CollectableRow) (variant, error) {
|
||||
var v variant
|
||||
return v, row.Scan(&v.Name, &v.ContentType, &v.Size, &v.Width, &v.Height, &v.SHA256, &v.key)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &x, nil
|
||||
}
|
||||
|
||||
func (a *App) attachmentFor(w http.ResponseWriter, r *http.Request) *attachment {
|
||||
reqID := reqIDFrom(r)
|
||||
u, _ := a.authUser(w, r)
|
||||
if u == nil {
|
||||
return nil
|
||||
}
|
||||
id := r.PathValue("attachment_id")
|
||||
if !isUUID(id) {
|
||||
a.writeErr(w, reqID, 400, "invalid_request", "invalid attachment_id")
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := dbCtx(r.Context())
|
||||
defer cancel()
|
||||
att, err := a.loadAttachment(ctx, id, u.ID)
|
||||
if err != nil {
|
||||
a.writeErr(w, reqID, 503, "unavailable", "database unavailable")
|
||||
return nil
|
||||
}
|
||||
if att == nil {
|
||||
a.writeErr(w, reqID, 404, "not_found", "attachment not found")
|
||||
return nil
|
||||
}
|
||||
return att
|
||||
}
|
||||
|
||||
func (a *App) handleGetAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
if att := a.attachmentFor(w, r); att != nil {
|
||||
a.writeJSON(w, 200, att.json())
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) handleDownloadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
reqID := reqIDFrom(r)
|
||||
name := r.URL.Query().Get("variant")
|
||||
if name == "" {
|
||||
name = "original"
|
||||
}
|
||||
if name != "original" && name != "thumbnail" && name != "crop" && name != "optimized" {
|
||||
a.writeErr(w, reqID, 400, "invalid_request", "invalid variant")
|
||||
return
|
||||
}
|
||||
att := a.attachmentFor(w, r)
|
||||
if att == nil {
|
||||
return
|
||||
}
|
||||
if att.Status != "ready" {
|
||||
a.writeErr(w, reqID, 409, "conflict", "attachment is not ready")
|
||||
return
|
||||
}
|
||||
key, contentType := att.Key, att.ContentType
|
||||
if name != "original" {
|
||||
key = ""
|
||||
for _, v := range att.Variants {
|
||||
if v.Name == name {
|
||||
key, contentType = v.key, v.ContentType
|
||||
}
|
||||
}
|
||||
if key == "" {
|
||||
a.writeErr(w, reqID, 404, "not_found", "variant not found")
|
||||
return
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
obj, info, err := a.store.Get(ctx, key)
|
||||
if err != nil {
|
||||
if isNoSuchKey(err) {
|
||||
a.writeErr(w, reqID, 404, "not_found", "attachment not found")
|
||||
return
|
||||
}
|
||||
a.log.Error("s3 get failed", "request_id", reqID, "err", err.Error())
|
||||
a.writeErr(w, reqID, 503, "unavailable", "storage unavailable")
|
||||
return
|
||||
}
|
||||
defer obj.Close()
|
||||
first := make([]byte, 32<<10)
|
||||
n, err := io.ReadFull(obj, first)
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
a.log.Error("s3 read failed", "request_id", reqID, "err", err.Error())
|
||||
a.writeErr(w, reqID, 503, "unavailable", "storage unavailable")
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(info.Size, 10))
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.WriteHeader(200)
|
||||
_, _ = w.Write(first[:n])
|
||||
_, _ = io.Copy(w, obj)
|
||||
}
|
||||
|
||||
var keepQueries = map[string]string{
|
||||
objectPrefix: `
|
||||
select k from unnest($1::text[]) k
|
||||
where exists(select 1 from attachments where object_key = k)
|
||||
or exists(select 1 from attachment_variants where object_key = k)`,
|
||||
stagingPrefix: `
|
||||
select k from unnest($1::text[]) k
|
||||
where exists(select 1 from attachments where staging_key = k and (status in ('queued', 'processing')
|
||||
or (status = 'failed' and updated_at > now() - make_interval(secs => $2))))`,
|
||||
}
|
||||
|
||||
func cleanupOrphans(ctx context.Context, db *pgxpool.Pool, store *Store, grace, retention time.Duration, log *slog.Logger) (int, error) {
|
||||
cutoff := time.Now().Add(-grace)
|
||||
removed := 0
|
||||
flush := func(prefix string, keys []string) error {
|
||||
if len(keys) == 0 {
|
||||
return nil
|
||||
}
|
||||
args := []any{keys}
|
||||
if prefix == stagingPrefix {
|
||||
args = append(args, retention.Seconds())
|
||||
}
|
||||
rows, err := db.Query(ctx, keepQueries[prefix], args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kept, err := pgx.CollectRows(rows, pgx.RowTo[string])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
known := map[string]bool{}
|
||||
for _, k := range kept {
|
||||
known[k] = true
|
||||
}
|
||||
for _, k := range keys {
|
||||
if known[k] {
|
||||
continue
|
||||
}
|
||||
if err := store.Remove(ctx, k); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Info("orphan object removed", "key", k)
|
||||
removed++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, prefix := range []string{objectPrefix, stagingPrefix} {
|
||||
var batch []string
|
||||
for obj := range store.List(ctx, prefix) {
|
||||
if obj.Err != nil {
|
||||
return removed, obj.Err
|
||||
}
|
||||
if obj.LastModified.After(cutoff) {
|
||||
continue
|
||||
}
|
||||
batch = append(batch, obj.Key)
|
||||
if len(batch) == 500 {
|
||||
if err := flush(prefix, batch); err != nil {
|
||||
return removed, err
|
||||
}
|
||||
batch = batch[:0]
|
||||
}
|
||||
}
|
||||
if err := flush(prefix, batch); err != nil {
|
||||
return removed, err
|
||||
}
|
||||
}
|
||||
return removed, nil
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/crypto/argon2"
|
||||
)
|
||||
|
||||
const (
|
||||
argonTime = 2
|
||||
argonMemory = 19456
|
||||
argonThreads = 1
|
||||
argonKeyLen = 32
|
||||
argonSaltLen = 16
|
||||
)
|
||||
|
||||
func hashPassword(password string) (string, error) {
|
||||
salt := make([]byte, argonSaltLen)
|
||||
if _, err := rand.Read(salt); err != nil {
|
||||
return "", err
|
||||
}
|
||||
hash := argon2.IDKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen)
|
||||
b64Salt := base64.RawStdEncoding.EncodeToString(salt)
|
||||
b64Hash := base64.RawStdEncoding.EncodeToString(hash)
|
||||
return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s",
|
||||
argon2.Version, argonMemory, argonTime, argonThreads, b64Salt, b64Hash), nil
|
||||
}
|
||||
|
||||
func checkPassword(encoded, password string) (bool, error) {
|
||||
parts := strings.Split(encoded, "$")
|
||||
if len(parts) != 6 || parts[1] != "argon2id" {
|
||||
return false, errors.New("bad hash")
|
||||
}
|
||||
var version int
|
||||
if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil {
|
||||
return false, err
|
||||
}
|
||||
var memory, timeCost, threads int
|
||||
for _, p := range strings.Split(parts[3], ",") {
|
||||
kv := strings.SplitN(p, "=", 2)
|
||||
if len(kv) != 2 {
|
||||
return false, errors.New("bad params")
|
||||
}
|
||||
n, err := strconv.Atoi(kv[1])
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
switch kv[0] {
|
||||
case "m":
|
||||
memory = n
|
||||
case "t":
|
||||
timeCost = n
|
||||
case "p":
|
||||
threads = n
|
||||
}
|
||||
}
|
||||
salt, err := base64.RawStdEncoding.DecodeString(parts[4])
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
want, err := base64.RawStdEncoding.DecodeString(parts[5])
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
got := argon2.IDKey([]byte(password), salt, uint32(timeCost), uint32(memory), uint8(threads), uint32(len(want)))
|
||||
return subtle.ConstantTimeCompare(got, want) == 1, nil
|
||||
}
|
||||
|
||||
func newToken() (string, []byte, string, error) {
|
||||
raw := make([]byte, 32)
|
||||
if _, err := rand.Read(raw); err != nil {
|
||||
return "", nil, "", err
|
||||
}
|
||||
token := base64.RawURLEncoding.EncodeToString(raw)
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
hash := make([]byte, len(sum))
|
||||
copy(hash, sum[:])
|
||||
return token, hash, hex.EncodeToString(hash), nil
|
||||
}
|
||||
|
||||
func tokenHash(token string) ([]byte, string) {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
hash := make([]byte, len(sum))
|
||||
copy(hash, sum[:])
|
||||
return hash, hex.EncodeToString(hash)
|
||||
}
|
||||
|
||||
func parseBasic(header string) (username, password string, ok bool) {
|
||||
const prefix = "Basic "
|
||||
if !strings.HasPrefix(header, prefix) {
|
||||
return "", "", false
|
||||
}
|
||||
decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(header[len(prefix):]))
|
||||
if err != nil {
|
||||
return "", "", false
|
||||
}
|
||||
s := string(decoded)
|
||||
i := strings.IndexByte(s, ':')
|
||||
if i < 0 {
|
||||
return "", "", false
|
||||
}
|
||||
username = s[:i]
|
||||
password = s[i+1:]
|
||||
return username, password, true
|
||||
}
|
||||
|
||||
func parseBearer(header string) (string, bool) {
|
||||
const prefix = "Bearer "
|
||||
if !strings.HasPrefix(header, prefix) {
|
||||
return "", false
|
||||
}
|
||||
tok := strings.TrimSpace(header[len(prefix):])
|
||||
if tok == "" {
|
||||
return "", false
|
||||
}
|
||||
return tok, true
|
||||
}
|
||||
|
||||
func validPassword(p string) bool {
|
||||
n := utf8.RuneCountInString(p)
|
||||
return n >= 12 && n <= 128
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
jobStream = "attachments.jobs"
|
||||
jobGroup = "workers"
|
||||
jobType = "attachment.process.v1"
|
||||
pipelineVersion = 1
|
||||
)
|
||||
|
||||
type jobMessage struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
JobID string `json:"job_id"`
|
||||
Type string `json:"type"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
}
|
||||
|
||||
type Broker struct {
|
||||
rdb *redis.Client
|
||||
}
|
||||
|
||||
func openBroker() (*Broker, error) {
|
||||
raw, err := readSecret("BROKER_URL", "BROKER_URL_FILE")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opt, err := redis.ParseURL(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opt.DialTimeout = 2 * time.Second
|
||||
opt.ReadTimeout = 3 * time.Second
|
||||
opt.WriteTimeout = 3 * time.Second
|
||||
opt.MaxRetries = 0
|
||||
return &Broker{rdb: redis.NewClient(opt)}, nil
|
||||
}
|
||||
|
||||
func (b *Broker) ensureGroup(ctx context.Context) error {
|
||||
err := b.rdb.XGroupCreateMkStream(ctx, jobStream, jobGroup, "0").Err()
|
||||
if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Broker) publish(ctx context.Context, m jobMessage) error {
|
||||
body, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return b.rdb.XAdd(ctx, &redis.XAddArgs{Stream: jobStream, Values: map[string]any{"job": string(body)}}).Err()
|
||||
}
|
||||
|
||||
func relayOnce(ctx context.Context, db *pgxpool.Pool, b *Broker) (int, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
tx, err := db.Begin(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
rows, err := tx.Query(ctx, `
|
||||
select o.id, o.job_id::text, o.attempt, j.attachment_id::text, j.pipeline_version, j.created_at
|
||||
from outbox o
|
||||
join jobs j on j.id = o.job_id
|
||||
where o.published_at is null and o.available_at <= now()
|
||||
order by o.id
|
||||
limit 50
|
||||
for update of o skip locked`)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var ids []int64
|
||||
var msgs []jobMessage
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var created time.Time
|
||||
m := jobMessage{SchemaVersion: 1, Type: jobType}
|
||||
if err := rows.Scan(&id, &m.JobID, &m.Attempt, &m.AttachmentID, &m.PipelineVersion, &created); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
m.CreatedAt = formatTime(created)
|
||||
ids = append(ids, id)
|
||||
msgs = append(msgs, m)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if err := b.ensureGroup(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, m := range msgs {
|
||||
if err := b.publish(ctx, m); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `update outbox set published_at = now() where id = any($1)`, ids); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(ids), tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func runRelay(ctx context.Context, db *pgxpool.Pool, b *Broker, log *slog.Logger) {
|
||||
for ctx.Err() == nil {
|
||||
n, err := relayOnce(ctx, db, b)
|
||||
if err != nil {
|
||||
log.Warn("outbox relay failed", "err", err.Error())
|
||||
time.Sleep(2 * time.Second)
|
||||
continue
|
||||
}
|
||||
if n > 0 {
|
||||
log.Info("outbox published", "count", n)
|
||||
continue
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
+321
@@ -0,0 +1,321 @@
|
||||
x-api-env: &api-env
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
SESSION_STORE: postgres
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-10}
|
||||
GOMEMLIMIT: 150MiB
|
||||
SHUTDOWN_TIMEOUT_SECONDS: 10
|
||||
DRAIN_SECONDS: 3
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
FAULT_POINT: ${API_FAULT_POINT:-}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
POSTGRES_DB: messenger
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- pg_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 5s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
networks:
|
||||
- backend
|
||||
restart: unless-stopped
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507
|
||||
command: ["valkey-server", "--appendonly", "yes", "--save", "60", "1"]
|
||||
volumes:
|
||||
- valkey_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 5s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
networks:
|
||||
- backend
|
||||
restart: unless-stopped
|
||||
|
||||
broker:
|
||||
image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507
|
||||
command: ["valkey-server", "/run/secrets/broker_conf", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory-policy", "noeviction"]
|
||||
secrets:
|
||||
- broker_conf
|
||||
volumes:
|
||||
- broker_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$(sed -n 's/^requirepass //p' /run/secrets/broker_conf)\" ping | grep -q PONG"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 5s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
networks:
|
||||
- backend
|
||||
restart: unless-stopped
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs@sha256:c2658820e763294a9e9de7b36c81e59b879e3459148351aee4e31a126ca64860
|
||||
command:
|
||||
- server
|
||||
- -dir=/data
|
||||
- -ip=s3
|
||||
- -master.volumeSizeLimitMB=128
|
||||
- -volume.max=30
|
||||
- -s3
|
||||
- -s3.port=9000
|
||||
- -s3.config=/run/secrets/s3_config
|
||||
secrets:
|
||||
- s3_config
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9333/dir/assign | grep -q fid"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
start_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
networks:
|
||||
- storage
|
||||
restart: unless-stopped
|
||||
|
||||
s3-init:
|
||||
image: mtusi-messenger:lab5
|
||||
command: ["s3-init"]
|
||||
environment:
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
|
||||
secrets:
|
||||
- s3_admin_access_key_id
|
||||
- s3_admin_secret_access_key
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
networks:
|
||||
- storage
|
||||
restart: "no"
|
||||
|
||||
janitor:
|
||||
image: mtusi-messenger:lab5
|
||||
command: ["janitor"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
CLEANUP_GRACE_SECONDS: ${CLEANUP_GRACE_SECONDS:-600}
|
||||
CLEANUP_INTERVAL_SECONDS: ${CLEANUP_INTERVAL_SECONDS:-300}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
disable: true
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 64M
|
||||
networks:
|
||||
- backend
|
||||
- storage
|
||||
restart: unless-stopped
|
||||
|
||||
worker:
|
||||
image: mtusi-messenger:lab5
|
||||
command: ["worker"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_BACKOFF_BASE_MS: ${JOB_BACKOFF_BASE_MS:-1000}
|
||||
JOB_BACKOFF_MAX_MS: ${JOB_BACKOFF_MAX_MS:-30000}
|
||||
GOMEMLIMIT: 600MiB
|
||||
FAULT_POINT: ${WORKER_FAULT_POINT:-}
|
||||
secrets:
|
||||
- database_url
|
||||
- broker_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
disable: true
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 768M
|
||||
networks:
|
||||
- backend
|
||||
- storage
|
||||
restart: unless-stopped
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
image: mtusi-messenger:lab5
|
||||
command: ["migrate"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets:
|
||||
- database_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
networks:
|
||||
- backend
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
image: mtusi-messenger:lab5
|
||||
environment: *api-env
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- broker_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 192M
|
||||
networks:
|
||||
- backend
|
||||
- storage
|
||||
restart: unless-stopped
|
||||
|
||||
lb:
|
||||
image: haproxy:3.0-alpine@sha256:cf3af0facf7eb22b9fee463daf0fbc07ead740b8150696dce724264faf7b4318
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
volumes:
|
||||
- ./lb/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
- api
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
networks:
|
||||
- backend
|
||||
restart: unless-stopped
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
broker_conf:
|
||||
file: ./secrets/broker.conf
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
s3_config:
|
||||
file: ./secrets/s3_config.json
|
||||
s3_admin_access_key_id:
|
||||
file: ./secrets/s3_admin_access_key_id
|
||||
s3_admin_secret_access_key:
|
||||
file: ./secrets/s3_admin_secret_access_key
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
|
||||
volumes:
|
||||
pg_data:
|
||||
valkey_data:
|
||||
broker_data:
|
||||
s3_data:
|
||||
|
||||
networks:
|
||||
backend:
|
||||
driver: bridge
|
||||
storage:
|
||||
driver: bridge
|
||||
@@ -0,0 +1,57 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func readSecret(envKey, fileKey string) (string, error) {
|
||||
if v := os.Getenv(envKey); v != "" {
|
||||
return strings.TrimSpace(v), nil
|
||||
}
|
||||
path := os.Getenv(fileKey)
|
||||
if path == "" {
|
||||
return "", fmt.Errorf("%s or %s required", envKey, fileKey)
|
||||
}
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(string(b)), nil
|
||||
}
|
||||
|
||||
func openDB(ctx context.Context) (*pgxpool.Pool, error) {
|
||||
dsn, err := readSecret("DATABASE_URL", "DATABASE_URL_FILE")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg, err := pgxpool.ParseConfig(dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.MaxConns = 10
|
||||
cfg.MinConns = 2
|
||||
cfg.MaxConnLifetime = 5 * time.Minute
|
||||
cfg.ConnConfig.ConnectTimeout = 3 * time.Second
|
||||
|
||||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
if err := pool.Ping(pingCtx); err != nil {
|
||||
pool.Close()
|
||||
return nil, err
|
||||
}
|
||||
return pool, nil
|
||||
}
|
||||
|
||||
func dbCtx(parent context.Context) (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(parent, 3*time.Second)
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
# 2026-11-05T21:56:06Z $ python3 tests/attachments.py
|
||||
PASS: attachment validation, limits and access control
|
||||
# exit=0
|
||||
@@ -0,0 +1,8 @@
|
||||
# 2026-11-05T21:57:18Z $ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 139, "failed_attempts": 1, "first_recovery_seconds": 3.475}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
# exit=0
|
||||
# api containers after hook
|
||||
mtusi2026lab5-fork-api-1 Exited (0) 12 seconds ago
|
||||
mtusi2026lab5-fork-api-2 Up 35 seconds (healthy)
|
||||
@@ -0,0 +1,7 @@
|
||||
# 2026-11-05T22:01:19Z $ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 140, "failed_attempts": 0, "first_recovery_seconds": 3.465}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
# api containers after hook
|
||||
mtusi2026lab5-fork-api-1 Up 35 seconds (healthy)
|
||||
mtusi2026lab5-fork-api-2 Exited (0) 12 seconds ago
|
||||
@@ -0,0 +1,71 @@
|
||||
{
|
||||
"baseline": {
|
||||
"window_seconds": 6,
|
||||
"successful_requests": 3146,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 524.3,
|
||||
"last_error_offset_seconds": null,
|
||||
"instances": [
|
||||
"4337e4656268",
|
||||
"eb2165af370d"
|
||||
]
|
||||
},
|
||||
"replicas": [
|
||||
{
|
||||
"stopped_replica": "eb2165af370d",
|
||||
"during_failure": {
|
||||
"window_seconds": 15.35,
|
||||
"successful_requests": 6738,
|
||||
"failed_attempts": 2,
|
||||
"successful_rps": 438.9,
|
||||
"last_error_offset_seconds": 0.66,
|
||||
"instances": [
|
||||
"4337e4656268",
|
||||
"eb2165af370d"
|
||||
]
|
||||
},
|
||||
"seen_stopped_replica_after_stop": false,
|
||||
"stop_command_seconds": 3.35,
|
||||
"restore_seconds": 6.48,
|
||||
"after_restore": {
|
||||
"window_seconds": 7.0,
|
||||
"successful_requests": 3915,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 559.3,
|
||||
"last_error_offset_seconds": null,
|
||||
"instances": [
|
||||
"4337e4656268",
|
||||
"eb2165af370d"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"stopped_replica": "4337e4656268",
|
||||
"during_failure": {
|
||||
"window_seconds": 15.5,
|
||||
"successful_requests": 6926,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 446.9,
|
||||
"last_error_offset_seconds": null,
|
||||
"instances": [
|
||||
"4337e4656268",
|
||||
"eb2165af370d"
|
||||
]
|
||||
},
|
||||
"seen_stopped_replica_after_stop": false,
|
||||
"stop_command_seconds": 3.5,
|
||||
"restore_seconds": 6.52,
|
||||
"after_restore": {
|
||||
"window_seconds": 7.0,
|
||||
"successful_requests": 3608,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 515.4,
|
||||
"last_error_offset_seconds": null,
|
||||
"instances": [
|
||||
"4337e4656268",
|
||||
"eb2165af370d"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# 2026-11-05T22:02:15Z $ python3 tests/faults.py
|
||||
1) api crashed after COMMIT before 202/publish: attachment=50f6ace7-1393-4c19-b47a-1def389a62c3 recovered by outbox relay (published rows=1) -> ready attempt=1
|
||||
2) worker crashed after S3 write before COMMIT: lease expired, reaper requeued, attempt=2 ready, variants=3
|
||||
3) worker crashed after COMMIT before ack: message redelivered after restart, skipped as no-op, attempt=1, variants=3
|
||||
4) s3 down: transient errors retried with backoff+jitter delays_ms=[931, 1284, 2760, 7484], then failed retry_exhausted, listed in registry
|
||||
replay: failed -> queued -> ready (audit rows=1); replay of ready job skipped, variants=3
|
||||
5) corrupted IDAT (passes header/tail check in API): failed invalid_image on attempt 1, no retries
|
||||
PASS: outbox recovery, lease recovery, redelivery after commit, backoff with jitter, retry_exhausted, replay, permanent error
|
||||
# exit=0
|
||||
@@ -0,0 +1,3 @@
|
||||
# 2026-11-05T21:56:12Z $ python3 tests/idempotency.py
|
||||
PASS: 32 parallel requests, 1 message, 2 instances
|
||||
# exit=0
|
||||
@@ -0,0 +1,25 @@
|
||||
# before
|
||||
api Up 2 minutes (healthy)
|
||||
api Up 2 minutes (healthy)
|
||||
broker Up 16 minutes (healthy)
|
||||
janitor Up 2 minutes
|
||||
lb Up 12 minutes
|
||||
postgres Up 19 minutes (healthy)
|
||||
s3 Up 7 minutes (healthy)
|
||||
valkey Up 19 minutes (healthy)
|
||||
worker Up 37 seconds
|
||||
# 2026-11-05T21:56:14Z $ make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
# exit=0
|
||||
# after
|
||||
api Up 2 seconds (health: starting)
|
||||
api Up 2 seconds (health: starting)
|
||||
broker Up 46 seconds (healthy)
|
||||
janitor Up 5 seconds
|
||||
lb Up 5 seconds
|
||||
postgres Up 46 seconds (healthy)
|
||||
s3 Up 36 seconds (healthy)
|
||||
valkey Up 46 seconds (healthy)
|
||||
worker Up Less than a second
|
||||
@@ -0,0 +1,7 @@
|
||||
# 2026-11-05T22:01:54Z $ python3 tests/queue_recovery.py
|
||||
accepted: attachment=d4c108ad-3da8-467c-aee0-eb2ee25759fc job=0208b98f-c201-4686-aa66-62b0bbf703ae status=queued, job in stream, worker stopped
|
||||
restarted broker and api: job still in stream, attachment still queued
|
||||
worker started: attachment ready, attempt=1, variants=3
|
||||
duplicate delivery: skipped as no-op, acked, variants=3, updated_at unchanged
|
||||
PASS: worker stop, broker/api restart, redelivery idempotency
|
||||
# exit=0
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"old_replicas": [
|
||||
"eb2165af370d",
|
||||
"4337e4656268"
|
||||
],
|
||||
"new_replicas": [
|
||||
"2ce132adb5c9",
|
||||
"2fe5ec734588"
|
||||
],
|
||||
"update_window": {
|
||||
"window_seconds": 16.4,
|
||||
"successful_requests": 10091,
|
||||
"failed_attempts": 4,
|
||||
"successful_rps": 615.4,
|
||||
"last_error_offset_seconds": 13.57,
|
||||
"instances": [
|
||||
"2ce132adb5c9",
|
||||
"2fe5ec734588",
|
||||
"4337e4656268",
|
||||
"eb2165af370d"
|
||||
]
|
||||
},
|
||||
"old_replicas_served_after_update": []
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
# 2026-11-05T22:04:42Z $ python3 tests/s3_checks.py
|
||||
PASS: private bucket, limited credentials, s3 outage, orphan and staging cleanup
|
||||
# exit=0
|
||||
@@ -0,0 +1 @@
|
||||
PASS: token revoked while valkey was stopped stays revoked after valkey restart (401)
|
||||
@@ -0,0 +1,6 @@
|
||||
# 2026-11-05T21:56:01Z $ sh -c make check && make test
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
# exit=0
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"file_bytes": 9722608,
|
||||
"concurrent_uploads": 12,
|
||||
"rounds": 3,
|
||||
"uploads_ok": 36,
|
||||
"uploads_failed": 0,
|
||||
"elapsed_seconds": 12.65,
|
||||
"api_memory_limit_mib": 192,
|
||||
"idle_memory_mib": {
|
||||
"mtusi2026lab5-fork-api-1": 62.4,
|
||||
"mtusi2026lab5-fork-api-2": 62.2
|
||||
},
|
||||
"peak_memory_mib": {
|
||||
"mtusi2026lab5-fork-api-1": 118.8,
|
||||
"mtusi2026lab5-fork-api-2": 99.5
|
||||
},
|
||||
"buffering_all_files_would_need_mib": 111.3
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
# 2026-11-05T21:56:02Z $ python3 tests/variants.py
|
||||
PASS: thumbnail/crop/optimized sizes, formats, checksums, no upscale, alpha on white, EXIF orientation, no EXIF in variants
|
||||
# exit=0
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"huge_images": {
|
||||
"count": 6,
|
||||
"pixels": 39942400,
|
||||
"bytes": 116545
|
||||
},
|
||||
"small_images": 20,
|
||||
"worker_concurrency": 2,
|
||||
"max_processing_jobs_observed": 2,
|
||||
"all_ready": true,
|
||||
"accept_seconds": 0.32,
|
||||
"all_ready_seconds": 13.1,
|
||||
"worker_peak_rss_mib": 616.7,
|
||||
"worker_memory_limit_mib": 768,
|
||||
"worker_restarts_oom": [
|
||||
"0",
|
||||
"false"
|
||||
],
|
||||
"timeout_case": {
|
||||
"job_timeout_seconds": 1,
|
||||
"status": "failed",
|
||||
"error_code": "retry_exhausted",
|
||||
"attempts": 5,
|
||||
"last_error": "context deadline exceeded"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
module mtusi-messenger
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/gen2brain/webp v0.6.4
|
||||
github.com/jackc/pgx/v5 v5.11.0
|
||||
github.com/minio/minio-go/v7 v7.4.1
|
||||
github.com/redis/go-redis/v9 v9.23.0
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
|
||||
golang.org/x/crypto v0.58.0
|
||||
golang.org/x/image v0.47.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ebitengine/purego v0.10.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.19.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.4 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.uber.org/atomic v1.12.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/net v0.60.0 // indirect
|
||||
golang.org/x/sync v0.24.0 // indirect
|
||||
golang.org/x/sys v0.49.0 // indirect
|
||||
golang.org/x/text v0.43.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.3 // indirect
|
||||
)
|
||||
@@ -0,0 +1,85 @@
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY=
|
||||
github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/gen2brain/webp v0.6.4 h1:SUDdmxADOAiPQ+5ylNmuHhuYf2dOi0KgKZHL5vpVCNU=
|
||||
github.com/gen2brain/webp v0.6.4/go.mod h1:iGWMaCSw7t3I/Cv9llzEKmpnR36S8lS8VL/ZVjxU0JE=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
|
||||
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w=
|
||||
github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=
|
||||
github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc=
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
|
||||
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE=
|
||||
go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||
golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo=
|
||||
golang.org/x/image v0.47.0/go.mod h1:IkN5o4A7vq55vXN8FWjjnhg9Rs2GOtffqsSi9mWAKYE=
|
||||
golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
|
||||
golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
|
||||
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
|
||||
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
|
||||
gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
+1051
File diff suppressed because it is too large
Load Diff
+176
@@ -0,0 +1,176 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"image"
|
||||
"image/draw"
|
||||
"image/jpeg"
|
||||
_ "image/png"
|
||||
|
||||
"github.com/gen2brain/webp"
|
||||
"github.com/rwcarlsen/goexif/exif"
|
||||
xdraw "golang.org/x/image/draw"
|
||||
)
|
||||
|
||||
const (
|
||||
jpegQuality = 85
|
||||
webpQuality = 80
|
||||
webpMethod = 4
|
||||
)
|
||||
|
||||
var errInvalidImage = errors.New("invalid image")
|
||||
|
||||
type variantSpec struct {
|
||||
name, contentType, ext string
|
||||
box int
|
||||
crop bool
|
||||
}
|
||||
|
||||
var variantSpecs = []variantSpec{
|
||||
{"thumbnail", "image/jpeg", "jpg", 256, false},
|
||||
{"crop", "image/jpeg", "jpg", 128, true},
|
||||
{"optimized", "image/webp", "webp", 1280, false},
|
||||
}
|
||||
|
||||
type rendered struct {
|
||||
spec variantSpec
|
||||
data []byte
|
||||
width, height int
|
||||
sha256 string
|
||||
}
|
||||
|
||||
func renderVariants(ctx context.Context, data []byte, maxPixels int64) ([]rendered, error) {
|
||||
cfg, format, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
if err != nil || cfg.Width < 1 || cfg.Height < 1 {
|
||||
return nil, errInvalidImage
|
||||
}
|
||||
if int64(cfg.Width)*int64(cfg.Height) > maxPixels {
|
||||
return nil, errInvalidImage
|
||||
}
|
||||
src, _, err := image.Decode(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, errInvalidImage
|
||||
}
|
||||
orient := 1
|
||||
if format == "jpeg" {
|
||||
orient = orientation(data)
|
||||
}
|
||||
src = shrink(src, 1280)
|
||||
var out []rendered
|
||||
for _, spec := range variantSpecs {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
img := render(src, cfg.Width, cfg.Height, orient, spec)
|
||||
var buf bytes.Buffer
|
||||
if spec.contentType == "image/webp" {
|
||||
err = webp.Encode(&buf, img, webp.Options{Quality: webpQuality, Method: webpMethod})
|
||||
} else {
|
||||
err = jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality})
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sum := sha256.Sum256(buf.Bytes())
|
||||
b := img.Bounds()
|
||||
out = append(out, rendered{spec: spec, data: buf.Bytes(), width: b.Dx(), height: b.Dy(), sha256: hex.EncodeToString(sum[:])})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func orientation(data []byte) int {
|
||||
x, err := exif.Decode(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return 1
|
||||
}
|
||||
tag, err := x.Get(exif.Orientation)
|
||||
if err != nil {
|
||||
return 1
|
||||
}
|
||||
v, err := tag.Int(0)
|
||||
if err != nil || v < 1 || v > 8 {
|
||||
return 1
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func fit(w, h, box int) (int, int) {
|
||||
if w <= box && h <= box {
|
||||
return w, h
|
||||
}
|
||||
if w >= h {
|
||||
return box, max(1, h*box/w)
|
||||
}
|
||||
return max(1, w*box/h), box
|
||||
}
|
||||
|
||||
func shrink(src image.Image, limit int) image.Image {
|
||||
for {
|
||||
b := src.Bounds()
|
||||
if max(b.Dx(), b.Dy())/2 < limit {
|
||||
return src
|
||||
}
|
||||
dst := image.NewRGBA(image.Rect(0, 0, max(1, b.Dx()/2), max(1, b.Dy()/2)))
|
||||
xdraw.ApproxBiLinear.Scale(dst, dst.Bounds(), src, b, xdraw.Src, nil)
|
||||
src = dst
|
||||
}
|
||||
}
|
||||
|
||||
func render(src image.Image, sw, sh, orient int, spec variantSpec) *image.RGBA {
|
||||
b := src.Bounds()
|
||||
rect := b
|
||||
var tw, th int
|
||||
if spec.crop {
|
||||
side := min(b.Dx(), b.Dy())
|
||||
x0, y0 := b.Min.X+(b.Dx()-side)/2, b.Min.Y+(b.Dy()-side)/2
|
||||
rect = image.Rect(x0, y0, x0+side, y0+side)
|
||||
tw, th = spec.box, spec.box
|
||||
} else if orient >= 5 {
|
||||
th, tw = fit(sh, sw, spec.box)
|
||||
} else {
|
||||
tw, th = fit(sw, sh, spec.box)
|
||||
}
|
||||
dst := image.NewRGBA(image.Rect(0, 0, tw, th))
|
||||
draw.Draw(dst, dst.Bounds(), image.White, image.Point{}, draw.Src)
|
||||
xdraw.CatmullRom.Scale(dst, dst.Bounds(), src, rect, xdraw.Over, nil)
|
||||
return orientImage(dst, orient)
|
||||
}
|
||||
|
||||
func orientImage(img *image.RGBA, o int) *image.RGBA {
|
||||
if o <= 1 {
|
||||
return img
|
||||
}
|
||||
w, h := img.Bounds().Dx(), img.Bounds().Dy()
|
||||
dw, dh := w, h
|
||||
if o >= 5 {
|
||||
dw, dh = h, w
|
||||
}
|
||||
dst := image.NewRGBA(image.Rect(0, 0, dw, dh))
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
var nx, ny int
|
||||
switch o {
|
||||
case 2:
|
||||
nx, ny = w-1-x, y
|
||||
case 3:
|
||||
nx, ny = w-1-x, h-1-y
|
||||
case 4:
|
||||
nx, ny = x, h-1-y
|
||||
case 5:
|
||||
nx, ny = y, x
|
||||
case 6:
|
||||
nx, ny = h-1-y, x
|
||||
case 7:
|
||||
nx, ny = h-1-y, w-1-x
|
||||
case 8:
|
||||
nx, ny = y, w-1-x
|
||||
}
|
||||
copy(dst.Pix[dst.PixOffset(nx, ny):dst.PixOffset(nx, ny)+4], img.Pix[img.PixOffset(x, y):img.PixOffset(x, y)+4])
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 2000
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
hold valid 1s
|
||||
hold nx 1s
|
||||
hold obsolete 1s
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option redispatch 1
|
||||
retries 3
|
||||
retry-on conn-failure empty-response
|
||||
timeout connect 1s
|
||||
timeout client 20s
|
||||
timeout server 15s
|
||||
timeout http-request 10s
|
||||
timeout queue 5s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr none on-marked-down shutdown-sessions
|
||||
server-template api 1-8 api:8080
|
||||
@@ -0,0 +1,196 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func main() {
|
||||
log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||
|
||||
host := envOr("APP_HOST", "0.0.0.0")
|
||||
port := envOr("APP_PORT", "8080")
|
||||
shutdownSec, _ := strconv.Atoi(envOr("SHUTDOWN_TIMEOUT_SECONDS", "5"))
|
||||
if shutdownSec < 1 {
|
||||
shutdownSec = 5
|
||||
}
|
||||
ttlSec, _ := strconv.Atoi(envOr("SESSION_TTL_SECONDS", "3600"))
|
||||
if ttlSec < 1 {
|
||||
ttlSec = 3600
|
||||
}
|
||||
|
||||
idemSec, _ := strconv.Atoi(envOr("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
if idemSec < 1 {
|
||||
idemSec = 86400
|
||||
}
|
||||
drainSec, _ := strconv.Atoi(envOr("DRAIN_SECONDS", "3"))
|
||||
|
||||
ctx := context.Background()
|
||||
if len(os.Args) > 1 && os.Args[1] == "s3-init" {
|
||||
if err := s3Init(ctx); err != nil {
|
||||
log.Error("s3 init failed", "err", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Info("bucket ready")
|
||||
return
|
||||
}
|
||||
db, err := openDB(ctx)
|
||||
if err != nil {
|
||||
log.Error("database open failed", "err", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if len(os.Args) > 1 && os.Args[1] == "migrate" {
|
||||
if err := migrate(ctx, db); err != nil {
|
||||
log.Error("migrate failed", "err", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Info("migrations applied")
|
||||
return
|
||||
}
|
||||
|
||||
store, err := openStore()
|
||||
if err != nil {
|
||||
log.Error("s3 open failed", "err", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if len(os.Args) > 1 && (os.Args[1] == "cleanup" || os.Args[1] == "janitor") {
|
||||
runCleanup(ctx, db, store, log, os.Args[1] == "janitor")
|
||||
return
|
||||
}
|
||||
|
||||
maxBytes, _ := strconv.ParseInt(envOr("MAX_UPLOAD_BYTES", "10485760"), 10, 64)
|
||||
maxPixels, _ := strconv.ParseInt(envOr("MAX_IMAGE_PIXELS", "40000000"), 10, 64)
|
||||
maxAttempts := envInt("JOB_MAX_ATTEMPTS", 5)
|
||||
|
||||
if len(os.Args) > 1 && (os.Args[1] == "replay" || os.Args[1] == "failed") {
|
||||
out := json.NewEncoder(os.Stdout)
|
||||
if os.Args[1] == "failed" {
|
||||
items, err := listFailed(ctx, db)
|
||||
if err != nil {
|
||||
log.Error("failed jobs query failed", "err", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, item := range items {
|
||||
_ = out.Encode(item)
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(os.Args) != 3 || !isUUID(os.Args[2]) {
|
||||
log.Error("usage: replay <attachment_id>")
|
||||
os.Exit(2)
|
||||
}
|
||||
result, err := replayJob(ctx, db, os.Args[2], maxAttempts)
|
||||
if err != nil {
|
||||
log.Error("replay failed", "err", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
_ = out.Encode(map[string]string{"attachment_id": os.Args[2], "result": result})
|
||||
return
|
||||
}
|
||||
|
||||
broker, err := openBroker()
|
||||
if err != nil {
|
||||
log.Error("broker open failed", "err", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if len(os.Args) > 1 && os.Args[1] == "worker" {
|
||||
timeout := time.Duration(envInt("JOB_TIMEOUT_SECONDS", 60)) * time.Second
|
||||
owner, _ := os.Hostname()
|
||||
w := &Worker{
|
||||
db: db, store: store, broker: broker, log: log,
|
||||
owner: owner + "-" + newUUID()[:8],
|
||||
timeout: timeout,
|
||||
lease: timeout + 15*time.Second,
|
||||
backoffBase: time.Duration(envInt("JOB_BACKOFF_BASE_MS", 1000)) * time.Millisecond,
|
||||
backoffMax: time.Duration(envInt("JOB_BACKOFF_MAX_MS", 30000)) * time.Millisecond,
|
||||
maxBytes: maxBytes,
|
||||
maxPixels: maxPixels,
|
||||
fault: os.Getenv("FAULT_POINT"),
|
||||
}
|
||||
runWorker(db, store, broker, log, w, envInt("WORKER_CONCURRENCY", 2))
|
||||
return
|
||||
}
|
||||
|
||||
slots := envInt("UPLOAD_CONCURRENCY", 10)
|
||||
app := NewApp(db, log, time.Duration(ttlSec)*time.Second, time.Duration(idemSec)*time.Second, store,
|
||||
limits{maxBytes: maxBytes, maxPixels: maxPixels, uploadSlots: slots, maxAttempts: maxAttempts})
|
||||
relayCtx, stopRelay := context.WithCancel(ctx)
|
||||
defer stopRelay()
|
||||
go runRelay(relayCtx, db, broker, log)
|
||||
|
||||
addr := net.JoinHostPort(host, port)
|
||||
srv := &http.Server{
|
||||
Addr: addr,
|
||||
Handler: app.Routes(),
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 10 * time.Second,
|
||||
WriteTimeout: 15 * time.Second,
|
||||
IdleTimeout: 30 * time.Second,
|
||||
}
|
||||
|
||||
go func() {
|
||||
log.Info("listening", "addr", addr, "env", envOr("APP_ENV", "lab"), "session_ttl_s", ttlSec)
|
||||
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
log.Error("server error", "err", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
}()
|
||||
|
||||
stop := make(chan os.Signal, 1)
|
||||
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
|
||||
sig := <-stop
|
||||
log.Info("shutdown started", "signal", sig.String())
|
||||
app.SetReady(false)
|
||||
time.Sleep(time.Duration(drainSec) * time.Second)
|
||||
|
||||
shCtx, cancel := context.WithTimeout(context.Background(), time.Duration(shutdownSec)*time.Second)
|
||||
defer cancel()
|
||||
if err := srv.Shutdown(shCtx); err != nil {
|
||||
log.Error("shutdown error", "err", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Info("shutdown finished")
|
||||
}
|
||||
|
||||
func runCleanup(ctx context.Context, db *pgxpool.Pool, store *Store, log *slog.Logger, loop bool) {
|
||||
graceSec, _ := strconv.Atoi(envOr("CLEANUP_GRACE_SECONDS", "600"))
|
||||
intervalSec, _ := strconv.Atoi(envOr("CLEANUP_INTERVAL_SECONDS", "300"))
|
||||
retention := time.Duration(envInt("STAGING_RETENTION_SECONDS", 86400)) * time.Second
|
||||
for {
|
||||
removed, err := cleanupOrphans(ctx, db, store, time.Duration(graceSec)*time.Second, retention, log)
|
||||
if err != nil {
|
||||
log.Error("cleanup failed", "err", err.Error())
|
||||
if !loop {
|
||||
os.Exit(1)
|
||||
}
|
||||
} else {
|
||||
log.Info("cleanup finished", "removed", removed)
|
||||
}
|
||||
if !loop {
|
||||
return
|
||||
}
|
||||
time.Sleep(time.Duration(intervalSec) * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func envInt(key string, def int) int {
|
||||
v, err := strconv.Atoi(os.Getenv(key))
|
||||
if err != nil || v < 1 {
|
||||
return def
|
||||
}
|
||||
return v
|
||||
}
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
//go:embed migrations/*.sql
|
||||
var migrationFS embed.FS
|
||||
|
||||
func migrate(ctx context.Context, pool *pgxpool.Pool) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
conn, err := pool.Acquire(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Release()
|
||||
if _, err := conn.Exec(ctx, `select pg_advisory_lock(727001)`); err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Exec(context.Background(), `select pg_advisory_unlock(727001)`)
|
||||
|
||||
_, err = conn.Exec(ctx, `
|
||||
create table if not exists schema_migrations (
|
||||
version text primary key,
|
||||
applied_at timestamptz not null
|
||||
)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("schema_migrations: %w", err)
|
||||
}
|
||||
|
||||
entries, err := migrationFS.ReadDir("migrations")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var names []string
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() && strings.HasSuffix(e.Name(), ".sql") {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
for _, name := range names {
|
||||
var exists bool
|
||||
err := conn.QueryRow(ctx, `select exists(select 1 from schema_migrations where version=$1)`, name).Scan(&exists)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if exists {
|
||||
continue
|
||||
}
|
||||
|
||||
body, err := migrationFS.ReadFile("migrations/" + name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tx, err := conn.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, string(body)); err != nil {
|
||||
_ = tx.Rollback(ctx)
|
||||
return fmt.Errorf("migration %s: %w", name, err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `insert into schema_migrations(version, applied_at) values($1, now())`, name); err != nil {
|
||||
_ = tx.Rollback(ctx)
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
create table users (
|
||||
id uuid primary key,
|
||||
username text not null unique,
|
||||
display_name text not null,
|
||||
password_hash text not null,
|
||||
created_at timestamptz not null
|
||||
);
|
||||
|
||||
create table chats (
|
||||
id uuid primary key,
|
||||
title text not null,
|
||||
created_at timestamptz not null
|
||||
);
|
||||
|
||||
create table chat_members (
|
||||
chat_id uuid not null references chats(id),
|
||||
user_id uuid not null references users(id),
|
||||
primary key (chat_id, user_id)
|
||||
);
|
||||
|
||||
create index chat_members_user_idx on chat_members (user_id);
|
||||
|
||||
create table messages (
|
||||
id uuid primary key,
|
||||
chat_id uuid not null references chats(id),
|
||||
sender_id uuid not null references users(id),
|
||||
text text not null,
|
||||
created_at timestamptz not null
|
||||
);
|
||||
|
||||
create index messages_history_idx on messages (chat_id, created_at, id);
|
||||
|
||||
create table sessions (
|
||||
token_hash bytea primary key,
|
||||
user_id uuid not null references users(id) on delete cascade,
|
||||
created_at timestamptz not null,
|
||||
expires_at timestamptz not null
|
||||
);
|
||||
@@ -0,0 +1,10 @@
|
||||
create table idempotency_keys (
|
||||
user_id uuid not null references users(id) on delete cascade,
|
||||
method text not null,
|
||||
path text not null,
|
||||
key text not null,
|
||||
body_hash text not null,
|
||||
response jsonb not null,
|
||||
created_at timestamptz not null,
|
||||
primary key (user_id, method, path, key)
|
||||
);
|
||||
@@ -0,0 +1,22 @@
|
||||
create table attachments (
|
||||
id uuid primary key,
|
||||
chat_id uuid not null references chats(id),
|
||||
owner_id uuid not null references users(id),
|
||||
filename text not null,
|
||||
content_type text not null,
|
||||
size_bytes bigint not null,
|
||||
sha256 text not null,
|
||||
object_key text not null unique,
|
||||
created_at timestamptz not null
|
||||
);
|
||||
|
||||
create index attachments_chat_idx on attachments (chat_id);
|
||||
|
||||
create table message_attachments (
|
||||
message_id uuid not null references messages(id),
|
||||
attachment_id uuid not null references attachments(id),
|
||||
position int not null,
|
||||
primary key (message_id, attachment_id)
|
||||
);
|
||||
|
||||
create index message_attachments_attachment_idx on message_attachments (attachment_id);
|
||||
@@ -0,0 +1,60 @@
|
||||
alter table attachments
|
||||
add column status text not null default 'ready',
|
||||
add column error_code text,
|
||||
add column staging_key text,
|
||||
add column updated_at timestamptz;
|
||||
|
||||
update attachments set updated_at = created_at;
|
||||
|
||||
alter table attachments
|
||||
alter column status drop default,
|
||||
alter column updated_at set not null,
|
||||
alter column object_key drop not null;
|
||||
|
||||
create table attachment_variants (
|
||||
attachment_id uuid not null references attachments(id),
|
||||
pipeline_version int not null,
|
||||
name text not null,
|
||||
object_key text not null unique,
|
||||
content_type text not null,
|
||||
size_bytes bigint not null,
|
||||
width int not null,
|
||||
height int not null,
|
||||
sha256 text not null,
|
||||
primary key (attachment_id, pipeline_version, name)
|
||||
);
|
||||
|
||||
create table jobs (
|
||||
id uuid primary key,
|
||||
attachment_id uuid not null unique references attachments(id),
|
||||
pipeline_version int not null,
|
||||
status text not null,
|
||||
attempt int not null,
|
||||
max_attempts int not null,
|
||||
next_attempt_at timestamptz not null,
|
||||
lease_owner text,
|
||||
lease_until timestamptz,
|
||||
last_error text,
|
||||
created_at timestamptz not null,
|
||||
updated_at timestamptz not null
|
||||
);
|
||||
|
||||
create index jobs_lease_idx on jobs (lease_until) where status = 'processing';
|
||||
|
||||
create table outbox (
|
||||
id bigserial primary key,
|
||||
job_id uuid not null references jobs(id),
|
||||
attempt int not null,
|
||||
available_at timestamptz not null,
|
||||
published_at timestamptz,
|
||||
created_at timestamptz not null
|
||||
);
|
||||
|
||||
create index outbox_pending_idx on outbox (available_at) where published_at is null;
|
||||
|
||||
create table job_replays (
|
||||
id bigserial primary key,
|
||||
job_id uuid not null references jobs(id),
|
||||
previous_error text,
|
||||
replayed_at timestamptz not null
|
||||
);
|
||||
@@ -0,0 +1,33 @@
|
||||
package main
|
||||
|
||||
import "time"
|
||||
|
||||
type User struct {
|
||||
ID string
|
||||
Username string
|
||||
DisplayName string
|
||||
PasswordHash string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type Chat struct {
|
||||
ID string
|
||||
Title string
|
||||
MemberIDs []string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type Message struct {
|
||||
ID string
|
||||
ChatID string
|
||||
SenderID string
|
||||
Text string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type Session struct {
|
||||
TokenHash []byte
|
||||
UserID string
|
||||
CreatedAt time.Time
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
func openStore() (*Store, error) {
|
||||
endpoint, err := url.Parse(envOr("S3_ENDPOINT", "http://s3:9000"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keyID, err := readSecret("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
secret, err := readSecret("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
transport := &http.Transport{
|
||||
DialContext: (&net.Dialer{Timeout: 2 * time.Second}).DialContext,
|
||||
ResponseHeaderTimeout: 10 * time.Second,
|
||||
MaxIdleConnsPerHost: 16,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
}
|
||||
client, err := minio.New(endpoint.Host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(keyID, secret, ""),
|
||||
Secure: endpoint.Scheme == "https",
|
||||
Region: envOr("S3_REGION", "us-east-1"),
|
||||
Transport: transport,
|
||||
MaxRetries: 1,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Store{client: client, bucket: envOr("S3_BUCKET", "messenger-attachments")}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Put(ctx context.Context, key string, r io.Reader, size int64, contentType string) error {
|
||||
_, err := s.client.PutObject(ctx, s.bucket, key, r, size, minio.PutObjectOptions{
|
||||
ContentType: contentType,
|
||||
PartSize: 5 << 20,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) Get(ctx context.Context, key string) (*minio.Object, minio.ObjectInfo, error) {
|
||||
obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, minio.ObjectInfo{}, err
|
||||
}
|
||||
info, err := obj.Stat()
|
||||
if err != nil {
|
||||
obj.Close()
|
||||
return nil, minio.ObjectInfo{}, err
|
||||
}
|
||||
return obj, info, nil
|
||||
}
|
||||
|
||||
func (s *Store) Remove(ctx context.Context, key string) error {
|
||||
return s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
func (s *Store) List(ctx context.Context, prefix string) <-chan minio.ObjectInfo {
|
||||
return s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true})
|
||||
}
|
||||
|
||||
func isNoSuchKey(err error) bool {
|
||||
return minio.ToErrorResponse(err).Code == "NoSuchKey"
|
||||
}
|
||||
|
||||
func s3Init(ctx context.Context) error {
|
||||
s, err := openStore()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
for {
|
||||
exists, err := s.client.BucketExists(ctx, s.bucket)
|
||||
if err == nil {
|
||||
if exists {
|
||||
return nil
|
||||
}
|
||||
err = s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: envOr("S3_REGION", "us-east-1")})
|
||||
if err == nil || minio.ToErrorResponse(err).Code == "BucketAlreadyOwnedByYou" {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return err
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
mkdir -p "$ROOT/secrets"
|
||||
cd "$ROOT/secrets"
|
||||
gen() {
|
||||
[ -f "$1" ] || openssl rand -hex "$2" | tr -d '\n' > "$1"
|
||||
}
|
||||
if [ ! -f postgres_password ]; then
|
||||
openssl rand -base64 24 | tr -d '\n' > postgres_password
|
||||
fi
|
||||
PASS=$(cat postgres_password)
|
||||
ENC=$(printf '%s' "$PASS" | python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.stdin.read(), safe=""))')
|
||||
printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$ENC" > database_url
|
||||
printf 'redis://valkey:6379/0\n' > cache_url
|
||||
gen broker_password 24
|
||||
printf 'requirepass %s\n' "$(cat broker_password)" > broker.conf
|
||||
printf 'redis://:%s@broker:6379/0\n' "$(cat broker_password)" > broker_url
|
||||
gen s3_admin_access_key_id 8
|
||||
gen s3_admin_secret_access_key 20
|
||||
gen s3_access_key_id 8
|
||||
gen s3_secret_access_key 20
|
||||
BUCKET=$(grep -E '^S3_BUCKET=' "$ROOT/.env" 2>/dev/null | cut -d= -f2 || true)
|
||||
BUCKET=${BUCKET:-messenger-attachments}
|
||||
printf '{"identities":[{"name":"admin","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write","List","Tagging"]},{"name":"api","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Read:%s","Write:%s","List:%s"]}]}\n' \
|
||||
"$(cat s3_admin_access_key_id)" "$(cat s3_admin_secret_access_key)" "$(cat s3_access_key_id)" "$(cat s3_secret_access_key)" "$BUCKET" "$BUCKET" "$BUCKET" > s3_config.json
|
||||
chmod 644 ./*
|
||||
echo "secrets ready in $ROOT/secrets"
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
docker compose --env-file .env restart postgres valkey broker s3
|
||||
docker compose --env-file .env up -d --wait postgres valkey broker s3
|
||||
docker compose --env-file .env restart api lb janitor worker
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
dc="docker compose --env-file .env"
|
||||
old=$($dc ps -q api)
|
||||
n=$(echo "$old" | wc -w)
|
||||
$dc build migrate
|
||||
$dc up -d --no-deps migrate janitor worker
|
||||
$dc up -d --no-deps --no-recreate --scale api=$((n * 2)) api
|
||||
$dc up -d --wait --no-deps --no-recreate --scale api=$((n * 2)) api
|
||||
for c in $old; do
|
||||
docker stop -t 10 "$c" >/dev/null
|
||||
docker rm "$c" >/dev/null
|
||||
done
|
||||
$dc up -d --wait --no-deps --no-recreate --scale api=$n api
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?}"
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
id=$(docker compose --env-file .env ps -q api | while read -r c; do
|
||||
case "$c" in "$TARGET_INSTANCE_ID"*) echo "$c";; esac
|
||||
done)
|
||||
[ -n "$id" ] || exit 1
|
||||
docker stop -t 10 "$id" >/dev/null
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from client import Client, arguments, check, run
|
||||
from image_gen import png, png_header_only
|
||||
from stack import wait_status
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
|
||||
|
||||
def upload(c, actor, chat, data, expected, content_type='image/png', filename='file.png'):
|
||||
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, expected,
|
||||
actor=actor, raw=data, headers={'Content-Type': content_type},
|
||||
schema='Attachment' if expected == 202 else None)
|
||||
|
||||
|
||||
def post_message(c, actor, chat, ids, expected, text=''):
|
||||
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', expected, actor=actor,
|
||||
body={'text': text, 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message' if expected == 201 else None)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Attachment validation and access checks')
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'second', 'member_ids': [alice['id'], bob['id']]}, schema='Chat')['json']
|
||||
eve_chat = c.call('POST', '/api/v1/chats', 201, actor=eve, body={'title': 'eve', 'member_ids': [eve['id'], bob['id']]}, schema='Chat')['json']
|
||||
|
||||
upload(c, alice, chat, b'x' * (MAX_BYTES + 1), 413)
|
||||
upload(c, alice, chat, SAMPLE + b'\0' * MAX_BYTES, 413)
|
||||
upload(c, alice, chat, png_header_only(8000, 8000), 413)
|
||||
upload(c, alice, chat, SAMPLE[:len(SAMPLE) // 2], 415)
|
||||
upload(c, alice, chat, SAMPLE[:-1], 415)
|
||||
upload(c, alice, chat, b'\x89PNG\r\n\x1a\n' + b'garbage' * 100, 415)
|
||||
upload(c, alice, chat, b'', 415)
|
||||
upload(c, alice, chat, b'GIF89a' + b'\0' * 64, 415)
|
||||
upload(c, alice, chat, SAMPLE, 415, content_type='image/jpeg')
|
||||
upload(c, alice, chat, SAMPLE, 415, content_type='text/plain')
|
||||
upload(c, alice, chat, SAMPLE, 415, content_type='application/octet-stream')
|
||||
|
||||
for name in ('..%2Fx.png', 'a%5Cb.png', 'a%00b.png', 'a%0Ab.png', 'a' * 256, ''):
|
||||
upload(c, alice, chat, SAMPLE, 400, filename=name)
|
||||
c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments', 400, actor=alice, raw=SAMPLE, headers={'Content-Type': 'image/png'})
|
||||
upload(c, eve, chat, SAMPLE, 404)
|
||||
c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=a.png', 401, raw=SAMPLE, headers={'Content-Type': 'image/png'})
|
||||
|
||||
ok = upload(c, alice, chat, SAMPLE, 202)['json']
|
||||
check(ok['sha256'] == hashlib.sha256(SAMPLE).hexdigest(), 'checksum mismatch')
|
||||
check(ok['filename'] == 'file.png' and ok['status'] == 'queued' and ok['variants'] == [], 'attachment fields')
|
||||
unicode_name = upload(c, alice, chat, SAMPLE, 202, filename='%D1%84%D0%BE%D1%82%D0%BE.png')['json']
|
||||
check(unicode_name['filename'] == 'фото.png', 'unicode filename changed')
|
||||
|
||||
post_message(c, alice, other, [ok['id']], 404)
|
||||
post_message(c, alice, chat, [str(uuid.uuid4())], 404)
|
||||
post_message(c, alice, chat, [ok['id'], ok['id']], 400)
|
||||
post_message(c, alice, chat, ['not-a-uuid'], 400)
|
||||
post_message(c, alice, chat, [str(uuid.uuid4()) for _ in range(11)], 400)
|
||||
post_message(c, alice, chat, [], 400)
|
||||
post_message(c, eve, chat, [ok['id']], 404)
|
||||
post_message(c, eve, eve_chat, [ok['id']], 404)
|
||||
|
||||
shared = post_message(c, bob, chat, [ok['id'], unicode_name['id']], 201, text='from bob')['json']
|
||||
check(shared['attachment_ids'] == [ok['id'], unicode_name['id']], 'attachment order changed')
|
||||
history = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=alice, schema='MessagePage')['json']['items']
|
||||
check(any(m['id'] == shared['id'] and m['attachment_ids'] == shared['attachment_ids'] for m in history), 'history lost attachment_ids')
|
||||
|
||||
for suffix in ('', '/content'):
|
||||
c.call('GET', '/api/v1/attachments/' + ok['id'] + suffix, 404, actor=eve)
|
||||
c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()) + suffix, 404, actor=alice)
|
||||
c.call('GET', '/api/v1/attachments/bad' + suffix, 400, actor=alice)
|
||||
c.call('GET', '/api/v1/attachments/' + ok['id'] + suffix, 401)
|
||||
c.call('GET', '/api/v1/attachments/' + ok['id'] + '/content?variant=bogus', 400, actor=bob)
|
||||
wait_status(c, bob, ok['id'], ('ready',), 60)
|
||||
download = c.call('GET', '/api/v1/attachments/' + ok['id'] + '/content', actor=bob)
|
||||
check(download['bytes'] == SAMPLE, 'download differs')
|
||||
|
||||
big = png(1800, 1800, noise=True)
|
||||
check(len(big) < MAX_BYTES, 'generated image exceeds limit')
|
||||
large = upload(c, alice, chat, big, 202)['json']
|
||||
wait_status(c, bob, large['id'], ('ready',), 60)
|
||||
check(c.call('GET', '/api/v1/attachments/' + large['id'] + '/content', actor=bob)['bytes'] == big, 'large download differs')
|
||||
print('PASS: attachment validation, limits and access control')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Executable
+157
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from client import Client, Failure, arguments, check, run
|
||||
from stack import (SAMPLE, broker, compose, job_row, logs, now_iso, psql, restore_api, restore_worker, upload,
|
||||
variant_count, wait_api, wait_log, wait_status)
|
||||
|
||||
|
||||
def job_id_of(att_id):
|
||||
return psql("select id from jobs where attachment_id='%s'" % att_id)
|
||||
|
||||
|
||||
def worker_with(fault, **extra):
|
||||
compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env=dict({'WORKER_FAULT_POINT': fault}, **extra))
|
||||
|
||||
|
||||
def wait_s3(timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while '(healthy)' not in compose('ps', 's3', '--format', '{{.Status}}').stdout:
|
||||
check(time.monotonic() < deadline, 's3 did not recover')
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def ops(*args):
|
||||
out = compose('run', '--rm', '--no-deps', '-T', 'worker', *args).stdout
|
||||
return [json.loads(line) for line in out.splitlines() if line.startswith('{')]
|
||||
|
||||
|
||||
def crash_after_commit(c, alice, bob, chat):
|
||||
since = now_iso()
|
||||
compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': 'after_commit'})
|
||||
compose('restart', 'lb')
|
||||
wait_api(c)
|
||||
name = 'crash-' + uuid.uuid4().hex[:12] + '.png'
|
||||
answered = None
|
||||
try:
|
||||
answered = upload(c, alice, chat, filename=name)
|
||||
except Failure:
|
||||
pass
|
||||
check(answered is None, 'api answered 202 despite injected crash')
|
||||
wait_log('api', since, '"point":"after_commit"', 20)
|
||||
att_id = psql("select id from attachments where filename='%s'" % name)
|
||||
check(att_id, 'committed attachment not found')
|
||||
restore_api(c)
|
||||
ready = wait_status(c, bob, att_id, ('ready', 'failed'), 60)
|
||||
check(ready['status'] == 'ready' and job_row(att_id)[:2] == ('ready', 1), 'outbox did not recover the job')
|
||||
published = psql("select count(*) from outbox where job_id='%s' and published_at is not null" % job_id_of(att_id))
|
||||
print('1) api crashed after COMMIT before 202/publish: attachment=%s recovered by outbox relay (published rows=%s) -> ready attempt=1'
|
||||
% (att_id, published))
|
||||
|
||||
|
||||
def crash_after_store(c, alice, bob, chat):
|
||||
compose('stop', 'worker')
|
||||
att = upload(c, alice, chat)['json']
|
||||
since = now_iso()
|
||||
worker_with('after_store', JOB_TIMEOUT_SECONDS='10')
|
||||
wait_log('worker', since, '"point":"after_store"', 60)
|
||||
status, attempt, _ = job_row(att['id'])
|
||||
check(status == 'processing' and attempt == 1, 'job must stay processing under expired-to-be lease')
|
||||
since = now_iso()
|
||||
worker_with('', JOB_TIMEOUT_SECONDS='10')
|
||||
wait_log('worker', since, '"lease expired","job_id":"%s"' % job_id_of(att['id']), 90)
|
||||
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60)
|
||||
status, attempt, _ = job_row(att['id'])
|
||||
check(ready['status'] == 'ready' and status == 'ready' and attempt >= 2, 'job did not finish after lease expiry')
|
||||
check(variant_count(att['id']) == 3 and len(ready['variants']) == 3, 'retry duplicated variants')
|
||||
thumb = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0]
|
||||
body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)['bytes']
|
||||
check(len(body) == thumb['size_bytes'], 'thumbnail differs from metadata')
|
||||
restore_worker()
|
||||
print('2) worker crashed after S3 write before COMMIT: lease expired, reaper requeued, attempt=%d ready, variants=3' % attempt)
|
||||
|
||||
|
||||
def crash_before_ack(c, alice, bob, chat):
|
||||
compose('stop', 'worker')
|
||||
att = upload(c, alice, chat)['json']
|
||||
job_id = job_id_of(att['id'])
|
||||
since = now_iso()
|
||||
worker_with('before_ack')
|
||||
wait_log('worker', since, '"point":"before_ack"', 60)
|
||||
wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 60)
|
||||
status, attempt, _ = job_row(att['id'])
|
||||
check(status == 'ready' and attempt == 1 and variant_count(att['id']) == 3, 'redelivery after commit changed result')
|
||||
check(broker('XPENDING', 'attachments.jobs', 'workers').splitlines()[0] == '0', 'redelivered message not acked')
|
||||
check(wait_status(c, bob, att['id'], ('ready',), 10)['status'] == 'ready', 'attachment not ready')
|
||||
restore_worker()
|
||||
print('3) worker crashed after COMMIT before ack: message redelivered after restart, skipped as no-op, attempt=1, variants=3')
|
||||
|
||||
|
||||
def retries_and_replay(c, alice, bob, chat):
|
||||
compose('stop', 'worker')
|
||||
att = upload(c, alice, chat)['json']
|
||||
job_id = job_id_of(att['id'])
|
||||
compose('stop', 's3')
|
||||
since = now_iso()
|
||||
try:
|
||||
compose('up', '-d', '--no-deps', 'worker')
|
||||
check('(healthy)' not in compose('ps', '-a', 's3', '--format', '{{.Status}}').stdout, 's3 must stay stopped')
|
||||
failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 120)
|
||||
check(failed['status'] == 'failed' and failed['error_code'] == 'retry_exhausted', 'expected retry_exhausted')
|
||||
text = logs('worker', since)
|
||||
delays = [int(d) for d in re.findall(r'"job retry scheduled","job_id":"%s".*?"delay_ms":(\d+)' % job_id, text)]
|
||||
check(len(delays) == 4, 'expected 4 scheduled retries, got %d' % len(delays))
|
||||
for n, d in enumerate(delays):
|
||||
base = 1000 * 2 ** n
|
||||
check(base // 2 <= d <= base, 'delay %d outside jittered window for retry %d' % (d, n + 1))
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob)
|
||||
c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, body={'text': '', 'attachment_ids': [att['id']]},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
check(any(x['attachment_id'] == att['id'] for x in ops('failed')), 'failed registry misses job')
|
||||
print('4) s3 down: transient errors retried with backoff+jitter delays_ms=%s, then failed retry_exhausted, listed in registry' % delays)
|
||||
finally:
|
||||
compose('start', 's3')
|
||||
wait_s3()
|
||||
check(ops('replay', att['id'])[0]['result'] == 'queued', 'replay did not queue job')
|
||||
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 120)
|
||||
check(ready['status'] == 'ready' and variant_count(att['id']) == 3, 'replayed job did not finish')
|
||||
check(ops('replay', att['id'])[0]['result'] == 'skipped_ready', 'replay of ready job must be a no-op')
|
||||
check(psql("select count(*) from job_replays where job_id='%s'" % job_id) == '1', 'replay audit missing')
|
||||
check(variant_count(att['id']) == 3, 'second replay created new effects')
|
||||
print(' replay: failed -> queued -> ready (audit rows=1); replay of ready job skipped, variants=3')
|
||||
|
||||
|
||||
def permanent_error(c, alice, bob, chat):
|
||||
data = bytearray(SAMPLE)
|
||||
mid = len(data) // 2
|
||||
data[mid:mid + 64] = b'\x00' * 64
|
||||
att = upload(c, alice, chat, bytes(data))['json']
|
||||
failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 60)
|
||||
status, attempt, _ = job_row(att['id'])
|
||||
check(failed['status'] == 'failed' and failed['error_code'] == 'invalid_image' and attempt == 1, 'corrupt image must fail without retries')
|
||||
check(failed['variants'] == [], 'failed attachment has variants')
|
||||
print('5) corrupted IDAT (passes header/tail check in API): failed invalid_image on attempt 1, no retries')
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Fault injection: COMMIT/publish gap, crash before COMMIT and before ack, retries, DLQ replay')
|
||||
c = Client(args.base_url, args.ca_file, timeout=15)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
try:
|
||||
crash_after_commit(c, alice, bob, chat)
|
||||
crash_after_store(c, alice, bob, chat)
|
||||
crash_before_ack(c, alice, bob, chat)
|
||||
retries_and_replay(c, alice, bob, chat)
|
||||
permanent_error(c, alice, bob, chat)
|
||||
finally:
|
||||
compose('up', '-d', '--no-deps', 's3')
|
||||
restore_worker()
|
||||
if 'after_commit' in compose('exec', '-T', 'api', 'printenv', 'FAULT_POINT', check_rc=False).stdout:
|
||||
restore_api(c)
|
||||
print('PASS: outbox recovery, lease recovery, redelivery after commit, backoff with jitter, retry_exhausted, replay, permanent error')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import threading
|
||||
import uuid
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Concurrent Idempotency-Key check through the load balancer')
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
||||
parallel = 32
|
||||
key = 'idem-' + uuid.uuid4().hex
|
||||
barrier = threading.Barrier(parallel)
|
||||
|
||||
def send(_):
|
||||
local = Client(args.base_url, args.ca_file)
|
||||
barrier.wait()
|
||||
return local.call('POST', path, 201, actor=alice, body={'text': 'same'}, headers={'Idempotency-Key': key}, schema='Message')
|
||||
|
||||
with ThreadPoolExecutor(max_workers=parallel) as pool:
|
||||
results = list(pool.map(send, range(parallel)))
|
||||
ids = {r['json']['id'] for r in results}
|
||||
instances = {r['headers']['X-Instance-Id'] for r in results}
|
||||
check(len(ids) == 1, 'parallel requests produced %d messages' % len(ids))
|
||||
check(len(instances) >= 2, 'requests were served by one instance only')
|
||||
check(len({r['json']['created_at'] for r in results}) == 1, 'created_at differs between replays')
|
||||
|
||||
listing = c.call('GET', path, actor=alice, schema='MessagePage')['json']['items']
|
||||
check(sum(1 for m in listing if m['text'] == 'same') == 1, 'duplicate message in history')
|
||||
|
||||
c.call('POST', path, 409, actor=alice, body={'text': 'other'}, headers={'Idempotency-Key': key})
|
||||
c.call('POST', path, 400, actor=alice, body={'text': 'x'})
|
||||
c.call('POST', path, 400, actor=alice, body={'text': 'x'}, headers={'Idempotency-Key': 'bad key!'})
|
||||
other = c.call('POST', path, 201, actor=bob, body={'text': 'same'}, headers={'Idempotency-Key': key}, schema='Message')['json']
|
||||
check(other['id'] not in ids, 'key must be scoped per user')
|
||||
print('PASS: %d parallel requests, 1 message, %d instances' % (parallel, len(instances)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,23 @@
|
||||
import os
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
|
||||
def chunk(kind, data):
|
||||
body = kind + data
|
||||
return struct.pack('>I', len(data)) + body + struct.pack('>I', zlib.crc32(body) & 0xffffffff)
|
||||
|
||||
|
||||
def png(width, height, noise=False):
|
||||
header = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0))
|
||||
raw = bytearray()
|
||||
row = width * 3 + 1
|
||||
for _ in range(height):
|
||||
raw += b'\x00' + (os.urandom(row - 1) if noise else bytes(row - 1))
|
||||
data = zlib.compress(bytes(raw), 0 if noise else 6)
|
||||
return header + chunk(b'IDAT', data) + chunk(b'IEND', b'')
|
||||
|
||||
|
||||
def png_header_only(width, height):
|
||||
header = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0))
|
||||
return header + chunk(b'IDAT', zlib.compress(b'\x00')) + chunk(b'IEND', b'')
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import time
|
||||
from client import Client, arguments, check, run
|
||||
from image_probe import jpeg_size
|
||||
from stack import broker, compose, job_row, logs, now_iso, psql, upload, variant_count, wait_api, wait_log, wait_status
|
||||
|
||||
|
||||
def stream_has(job_id):
|
||||
return job_id in broker('XRANGE', 'attachments.jobs', '-', '+')
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Worker stop, API/broker restart and redelivery scenario')
|
||||
c = Client(args.base_url, args.ca_file, timeout=15)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
att = upload(c, alice, chat)['json']
|
||||
check(att['status'] == 'queued' and att['variants'] == [], 'upload must return queued snapshot')
|
||||
msg = c.message(alice, chat, '', attachments=[att['id']])['json']
|
||||
check(msg['attachment_ids'] == [att['id']], 'message lost queued attachment')
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob)
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 409, actor=bob)
|
||||
c.message(bob, chat, 'text works while worker is down')
|
||||
job_id = psql("select id from jobs where attachment_id='%s'" % att['id'])
|
||||
deadline = time.monotonic() + 10
|
||||
while not stream_has(job_id):
|
||||
check(time.monotonic() < deadline, 'job was not published to broker')
|
||||
time.sleep(0.5)
|
||||
check(psql("select count(*) from outbox where job_id='%s' and published_at is null" % job_id) == '0', 'outbox not marked')
|
||||
print('accepted: attachment=%s job=%s status=queued, job in stream, worker stopped' % (att['id'], job_id))
|
||||
|
||||
compose('restart', 'broker', 'api')
|
||||
compose('up', '-d', '--wait', '--no-deps', 'broker', 'api')
|
||||
compose('restart', 'lb')
|
||||
wait_api(c)
|
||||
check(stream_has(job_id), 'job lost after broker restart')
|
||||
check(c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status'] == 'queued', 'status changed without worker')
|
||||
print('restarted broker and api: job still in stream, attachment still queued')
|
||||
finally:
|
||||
compose('start', 'worker')
|
||||
|
||||
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60)
|
||||
check(ready['status'] == 'ready', 'job did not finish after worker start')
|
||||
thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)
|
||||
check(jpeg_size(thumb['bytes']) == (256, 160), 'thumbnail size')
|
||||
check(job_row(att['id'])[:2] == ('ready', 1) and variant_count(att['id']) == 3, 'unexpected job state or variants')
|
||||
print('worker started: attachment ready, attempt=1, variants=3')
|
||||
|
||||
since = now_iso()
|
||||
before = psql("select updated_at from attachments where id='%s'" % att['id'])
|
||||
payload = json.dumps({'schema_version': 1, 'job_id': job_id, 'type': 'attachment.process.v1', 'attachment_id': att['id'],
|
||||
'pipeline_version': 1, 'attempt': 1, 'created_at': ready['created_at']})
|
||||
broker('XADD', 'attachments.jobs', '*', 'job', payload)
|
||||
wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 30)
|
||||
time.sleep(1)
|
||||
check(psql("select updated_at from attachments where id='%s'" % att['id']) == before, 'duplicate delivery changed attachment')
|
||||
check(variant_count(att['id']) == 3 and job_row(att['id'])[:2] == ('ready', 1), 'duplicate delivery created new effects')
|
||||
pending = broker('XPENDING', 'attachments.jobs', 'workers')
|
||||
check(pending.splitlines()[0] == '0', 'duplicate message was not acked')
|
||||
print('duplicate delivery: skipped as no-op, acked, variants=3, updated_at unchanged')
|
||||
print('PASS: worker stop, broker/api restart, redelivery idempotency')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from client import Client, Failure, arguments, check, run
|
||||
from stack import wait_status
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
BUCKET = 'messenger-attachments'
|
||||
|
||||
|
||||
def compose(*args, check_rc=True, stdin=None):
|
||||
return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=check_rc,
|
||||
capture_output=True, text=True, input=stdin)
|
||||
|
||||
|
||||
def secret(name):
|
||||
return (ROOT / 'secrets' / name).read_text().strip()
|
||||
|
||||
|
||||
def s3_status(method, path, user=None, data=None):
|
||||
cmd = ['exec', '-T', 's3', 'curl', '-s', '-o', '/dev/null', '-w', '%{http_code}', '-X', method]
|
||||
if user:
|
||||
cmd += ['--aws-sigv4', 'aws:amz:us-east-1:s3', '--user', user[0] + ':' + user[1]]
|
||||
if data is not None:
|
||||
cmd += ['--data-binary', data]
|
||||
cmd.append('http://127.0.0.1:9000' + path)
|
||||
return compose(*cmd).stdout.strip()
|
||||
|
||||
|
||||
def upload(c, actor, chat, expected):
|
||||
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=a.png', expected, actor=actor,
|
||||
raw=SAMPLE, headers={'Content-Type': 'image/png'}, schema='Attachment' if expected == 202 else None)
|
||||
|
||||
|
||||
def wait_api(c):
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call('GET', '/health/ready')
|
||||
return
|
||||
except Failure:
|
||||
time.sleep(1)
|
||||
raise Failure('api did not recover')
|
||||
|
||||
|
||||
def wait_s3(timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if compose('ps', 's3', '--format', '{{.Status}}').stdout.count('(healthy)'):
|
||||
return
|
||||
time.sleep(1)
|
||||
raise Failure('s3 did not recover')
|
||||
|
||||
|
||||
def object_key(attachment_id):
|
||||
out = compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc',
|
||||
"select object_key from attachments where id='%s'" % attachment_id).stdout.strip()
|
||||
check(out.startswith('attachments/'), 'object key not found')
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('S3 privacy, outage and orphan cleanup checks')
|
||||
c = Client(args.base_url, args.ca_file, timeout=15)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
att = upload(c, alice, chat, 202)['json']
|
||||
wait_status(c, bob, att['id'], ('ready',), 60)
|
||||
key = object_key(att['id'])
|
||||
check(key != 'attachments/' + att['filename'] and att['filename'] not in key, 'object key derived from filename')
|
||||
|
||||
api = (secret('s3_access_key_id'), secret('s3_secret_access_key'))
|
||||
admin = (secret('s3_admin_access_key_id'), secret('s3_admin_secret_access_key'))
|
||||
path = '/' + BUCKET + '/' + key
|
||||
check(s3_status('GET', path) == '403', 'anonymous GET must be denied')
|
||||
check(s3_status('GET', '/' + BUCKET + '/') == '403', 'anonymous listing must be denied')
|
||||
check(s3_status('PUT', path + '-anon', data='x') == '403', 'anonymous PUT must be denied')
|
||||
check(s3_status('GET', path, ('wrong', 'wrong')) == '403', 'wrong credentials must be denied')
|
||||
check(s3_status('GET', path, api) == '200', 'api credentials cannot read')
|
||||
check(s3_status('PUT', '/other-bucket-' + uuid.uuid4().hex[:8], api) == '403', 'api credentials can create buckets')
|
||||
ports = compose('ps', '--format', '{{.Service}} {{.Ports}}').stdout
|
||||
published = [line.split()[0] for line in ports.splitlines() if '->' in line]
|
||||
check(published == ['lb'], 'only lb may publish ports, found %s' % published)
|
||||
|
||||
compose('stop', 's3')
|
||||
try:
|
||||
upload(c, alice, chat, 503)
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob)
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'], 200, actor=bob, schema='Attachment')
|
||||
c.message(alice, chat, 'text works while s3 is down')
|
||||
finally:
|
||||
compose('start', 's3')
|
||||
wait_s3()
|
||||
wait_api(c)
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes']
|
||||
check(body == SAMPLE, 'content differs after s3 restart')
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic() < deadline, 'download did not recover')
|
||||
time.sleep(1)
|
||||
upload(c, alice, chat, 202)
|
||||
|
||||
orphan = '/' + BUCKET + '/attachments/' + uuid.uuid4().hex
|
||||
check(s3_status('PUT', orphan, admin, 'orphan') == '200', 'cannot create orphan')
|
||||
compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=3600', 'janitor', 'cleanup')
|
||||
check(s3_status('GET', orphan, admin) == '200', 'fresh object was removed inside grace period')
|
||||
time.sleep(1)
|
||||
compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=0', 'janitor', 'cleanup')
|
||||
check(s3_status('GET', orphan, admin) == '404', 'orphan was not removed')
|
||||
check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == SAMPLE, 'referenced object was removed')
|
||||
|
||||
check(s3_status('GET', '/' + BUCKET + '/staging/' + att['id'], admin) == '404', 'staging kept after ready')
|
||||
stale = '/' + BUCKET + '/staging/' + str(uuid.uuid4())
|
||||
check(s3_status('PUT', stale, admin, 'stale') == '200', 'cannot create stale staging object')
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
queued = upload(c, alice, chat, 202)['json']
|
||||
compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=0', 'janitor', 'cleanup')
|
||||
check(s3_status('GET', '/' + BUCKET + '/staging/' + queued['id'], admin) == '200', 'staging of queued job was removed')
|
||||
check(s3_status('GET', stale, admin) == '404', 'unreferenced staging object was not removed')
|
||||
finally:
|
||||
compose('up', '-d', '--no-deps', 'worker')
|
||||
check(wait_status(c, bob, queued['id'], ('ready', 'failed'), 60)['status'] == 'ready', 'queued job lost staging')
|
||||
print('PASS: private bucket, limited credentials, s3 outage, orphan and staging cleanup')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def sh(*cmd):
|
||||
return subprocess.run(cmd, cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def api_containers():
|
||||
out = sh('docker', 'compose', '--env-file', '.env', 'ps', '-q', 'api')
|
||||
return [x[:12] for x in out.split()]
|
||||
|
||||
|
||||
def wait_healthy(count, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
out = sh('docker', 'compose', '--env-file', '.env', 'ps', 'api', '--format', '{{.Status}}')
|
||||
if out.count('(healthy)') >= count:
|
||||
return
|
||||
time.sleep(1)
|
||||
raise Failure('replicas did not become healthy')
|
||||
|
||||
|
||||
class Load:
|
||||
def __init__(self, base, ca, concurrency):
|
||||
c = Client(base, ca)
|
||||
self.alice, _, _, self.chat = c.bootstrap()
|
||||
self.base, self.ca, self.n = base, ca, concurrency
|
||||
self.events = []
|
||||
self.lock = threading.Lock()
|
||||
self.stop = threading.Event()
|
||||
self.t0 = time.monotonic()
|
||||
|
||||
def worker(self):
|
||||
c = Client(self.base, self.ca, timeout=3)
|
||||
while not self.stop.is_set():
|
||||
key = uuid.uuid4().hex
|
||||
first_error = None
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
r = c.message(self.alice, self.chat, 'k-' + key, key=key)
|
||||
t = time.monotonic() - self.t0
|
||||
with self.lock:
|
||||
self.events.append((t, True, r['headers']['X-Instance-Id'], first_error))
|
||||
break
|
||||
except Failure:
|
||||
t = time.monotonic() - self.t0
|
||||
first_error = first_error or t
|
||||
with self.lock:
|
||||
self.events.append((t, False, None, None))
|
||||
time.sleep(0.05)
|
||||
|
||||
def start(self):
|
||||
self.pool = ThreadPoolExecutor(max_workers=self.n)
|
||||
self.futures = [self.pool.submit(self.worker) for _ in range(self.n)]
|
||||
|
||||
def finish(self):
|
||||
self.stop.set()
|
||||
self.pool.shutdown(wait=True)
|
||||
|
||||
def summary(self, begin, end):
|
||||
ev = [e for e in self.events if begin <= e[0] < end]
|
||||
ok = [e for e in ev if e[1]]
|
||||
bad = [e for e in ev if not e[1]]
|
||||
span = max(end - begin, 1e-9)
|
||||
return {
|
||||
'window_seconds': round(span, 2),
|
||||
'successful_requests': len(ok),
|
||||
'failed_attempts': len(bad),
|
||||
'successful_rps': round(len(ok) / span, 1),
|
||||
'last_error_offset_seconds': round(max(e[0] for e in bad) - begin, 2) if bad else None,
|
||||
'instances': sorted({e[2] for e in ok}),
|
||||
}
|
||||
|
||||
def instances_between(self, begin, end=float('inf')):
|
||||
return sorted({e[2] for e in self.events if e[1] and begin <= e[0] < end})
|
||||
|
||||
|
||||
def failover_each(args):
|
||||
ids = api_containers()
|
||||
check(len(ids) >= 2, 'need at least two replicas')
|
||||
load = Load(args.base_url, args.ca_file, args.concurrency)
|
||||
load.start()
|
||||
time.sleep(args.warmup)
|
||||
results = []
|
||||
base = load.summary(2, args.warmup)
|
||||
for target in ids:
|
||||
before = time.monotonic() - load.t0
|
||||
sh('docker', 'stop', '-t', '10', target)
|
||||
stopped = time.monotonic() - load.t0
|
||||
time.sleep(args.hold)
|
||||
after_stop = time.monotonic() - load.t0
|
||||
window = load.summary(before, after_stop)
|
||||
sh('docker', 'start', target)
|
||||
wait_healthy(len(ids))
|
||||
restored = time.monotonic() - load.t0
|
||||
time.sleep(args.warmup)
|
||||
results.append({
|
||||
'stopped_replica': target,
|
||||
'during_failure': window,
|
||||
'seen_stopped_replica_after_stop': target in load.instances_between(stopped + 0.5, after_stop),
|
||||
'stop_command_seconds': round(stopped - before, 2),
|
||||
'restore_seconds': round(restored - after_stop, 2),
|
||||
'after_restore': load.summary(restored + 1, restored + args.warmup),
|
||||
})
|
||||
load.finish()
|
||||
return {'baseline': base, 'replicas': results}
|
||||
|
||||
|
||||
def rolling(args):
|
||||
old = api_containers()
|
||||
check(len(old) >= 2, 'need at least two replicas')
|
||||
n = len(old)
|
||||
load = Load(args.base_url, args.ca_file, args.concurrency)
|
||||
load.start()
|
||||
time.sleep(args.warmup)
|
||||
begin = time.monotonic() - load.t0
|
||||
sh('scripts/rolling-update.sh')
|
||||
end = time.monotonic() - load.t0
|
||||
time.sleep(args.warmup)
|
||||
load.finish()
|
||||
return {
|
||||
'old_replicas': old,
|
||||
'new_replicas': api_containers(),
|
||||
'update_window': load.summary(begin, end),
|
||||
'old_replicas_served_after_update': [i for i in load.instances_between(end) if i in old],
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
def configure(p):
|
||||
p.add_argument('--mode', choices=['failover-each', 'rolling'], required=True)
|
||||
p.add_argument('--concurrency', type=int, default=8)
|
||||
p.add_argument('--warmup', type=float, default=8)
|
||||
p.add_argument('--hold', type=float, default=12)
|
||||
p.add_argument('--output')
|
||||
args = arguments('Failure and rolling update under load', configure)
|
||||
report = failover_each(args) if args.mode == 'failover-each' else rolling(args)
|
||||
text = json.dumps(report, indent=2)
|
||||
if args.output:
|
||||
Path(args.output).write_text(text + '\n')
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
import time
|
||||
from client import Client, arguments, check, run
|
||||
from stack import compose
|
||||
|
||||
|
||||
def wait_valkey(timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while '(healthy)' not in compose('ps', 'valkey', '--format', '{{.Status}}').stdout:
|
||||
check(time.monotonic() < deadline, 'valkey did not recover')
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Session revocation must not depend on the cache')
|
||||
c = Client(args.base_url, args.ca_file, timeout=15)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
c.call('GET', '/api/v1/chats', 200, actor=alice)
|
||||
compose('stop', 'valkey')
|
||||
try:
|
||||
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
|
||||
finally:
|
||||
compose('start', 'valkey')
|
||||
wait_valkey()
|
||||
c.call('GET', '/api/v1/chats', 401, actor=alice)
|
||||
c.call('GET', '/api/v1/chats', 200, actor=bob)
|
||||
print('PASS: token revoked while valkey was stopped stays revoked after valkey restart (401)')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,92 @@
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from client import Failure, check
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
|
||||
|
||||
def compose(*args, env=None, check_rc=True):
|
||||
full = dict(os.environ, **(env or {}))
|
||||
return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=check_rc,
|
||||
capture_output=True, text=True, env=full)
|
||||
|
||||
|
||||
def psql(sql):
|
||||
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', sql).stdout.strip()
|
||||
|
||||
|
||||
def broker(*args):
|
||||
cmd = 'valkey-cli --no-auth-warning -a "$(sed -n "s/^requirepass //p" /run/secrets/broker_conf)" ' + ' '.join("'%s'" % a for a in args)
|
||||
return compose('exec', '-T', 'broker', 'sh', '-c', cmd).stdout.strip()
|
||||
|
||||
|
||||
def logs(service, since):
|
||||
return compose('logs', '--no-color', '--since', since, service).stdout
|
||||
|
||||
|
||||
def now_iso():
|
||||
return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
||||
|
||||
|
||||
def upload(c, actor, chat, data=SAMPLE, content_type='image/png', filename=None, expected=202):
|
||||
filename = filename or 'f-' + uuid.uuid4().hex[:12] + '.png'
|
||||
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, expected, actor=actor,
|
||||
raw=data, headers={'Content-Type': content_type}, schema='Attachment' if expected == 202 else None)
|
||||
|
||||
|
||||
def wait_status(c, actor, att_id, statuses, timeout=120):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
att = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json']
|
||||
if att['status'] in statuses:
|
||||
return att
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise Failure('attachment %s did not reach %s' % (att_id, statuses))
|
||||
|
||||
|
||||
def wait_api(c, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call('GET', '/health/ready')
|
||||
return
|
||||
except Failure:
|
||||
time.sleep(1)
|
||||
raise Failure('api did not become ready')
|
||||
|
||||
|
||||
def wait_log(service, since, needle, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if needle in logs(service, since):
|
||||
return
|
||||
time.sleep(1)
|
||||
raise Failure('%s did not log %r' % (service, needle))
|
||||
|
||||
|
||||
def job_row(att_id):
|
||||
out = psql("select status, attempt, coalesce(last_error, '') from jobs where attachment_id='%s'" % att_id)
|
||||
check(out, 'job not found')
|
||||
status, attempt, error = out.split('|', 2)
|
||||
return status, int(attempt), error
|
||||
|
||||
|
||||
def variant_count(att_id):
|
||||
return int(psql("select count(*) from attachment_variants where attachment_id='%s'" % att_id))
|
||||
|
||||
|
||||
def restore_worker():
|
||||
compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'WORKER_FAULT_POINT': ''})
|
||||
|
||||
|
||||
def restore_api(c):
|
||||
compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': ''})
|
||||
compose('restart', 'lb')
|
||||
wait_api(c)
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from client import Client, Failure, arguments, check, run
|
||||
from image_gen import png
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
UNITS = {'B': 1, 'KiB': 1024, 'MiB': 1024 ** 2, 'GiB': 1024 ** 3}
|
||||
|
||||
|
||||
def to_mib(text):
|
||||
value, unit = re.match(r'([\d.]+)\s*([A-Za-z]+)', text).groups()
|
||||
return float(value) * UNITS[unit] / 1024 ** 2
|
||||
|
||||
|
||||
def sample(stop, peaks):
|
||||
proc = subprocess.Popen(['docker', 'stats', '--format', '{{.Name}} {{.MemUsage}}'], cwd=ROOT,
|
||||
stdout=subprocess.PIPE, text=True)
|
||||
try:
|
||||
for line in proc.stdout:
|
||||
line = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line).strip()
|
||||
m = re.match(r'(\S+-api-\d+)\s+(\S+ ?\S+) /', line)
|
||||
if m:
|
||||
peaks[m.group(1)] = max(peaks.get(m.group(1), 0), to_mib(m.group(2)))
|
||||
if stop.is_set():
|
||||
break
|
||||
finally:
|
||||
proc.terminate()
|
||||
|
||||
|
||||
def main():
|
||||
def configure(p):
|
||||
p.add_argument('--concurrency', type=int, default=12)
|
||||
p.add_argument('--rounds', type=int, default=3)
|
||||
p.add_argument('--output')
|
||||
args = arguments('Concurrent upload memory experiment', configure)
|
||||
check(args.concurrency >= 10, 'use at least 10 concurrent uploads')
|
||||
c = Client(args.base_url, args.ca_file, timeout=60)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
data = png(1800, 1800, noise=True)
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=load.png'
|
||||
|
||||
idle, stop = {}, threading.Event()
|
||||
t = threading.Thread(target=sample, args=(stop, idle))
|
||||
t.start()
|
||||
time.sleep(4)
|
||||
stop.set()
|
||||
t.join()
|
||||
|
||||
peaks, stop = {}, threading.Event()
|
||||
t = threading.Thread(target=sample, args=(stop, peaks))
|
||||
t.start()
|
||||
time.sleep(2)
|
||||
barrier = threading.Barrier(args.concurrency)
|
||||
results = []
|
||||
|
||||
def worker(_):
|
||||
local = Client(args.base_url, args.ca_file, timeout=60)
|
||||
barrier.wait()
|
||||
out = []
|
||||
for _ in range(args.rounds):
|
||||
try:
|
||||
r = local.call('POST', path, 202, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
out.append(r['json']['sha256'] == digest)
|
||||
except Failure:
|
||||
out.append(False)
|
||||
return out
|
||||
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
for out in pool.map(worker, range(args.concurrency)):
|
||||
results.extend(out)
|
||||
elapsed = time.monotonic() - started
|
||||
time.sleep(2)
|
||||
stop.set()
|
||||
t.join()
|
||||
limit = 192
|
||||
report = {
|
||||
'file_bytes': len(data),
|
||||
'concurrent_uploads': args.concurrency,
|
||||
'rounds': args.rounds,
|
||||
'uploads_ok': sum(results),
|
||||
'uploads_failed': len(results) - sum(results),
|
||||
'elapsed_seconds': round(elapsed, 2),
|
||||
'api_memory_limit_mib': limit,
|
||||
'idle_memory_mib': {k: round(v, 1) for k, v in sorted(idle.items())},
|
||||
'peak_memory_mib': {k: round(v, 1) for k, v in sorted(peaks.items())},
|
||||
'buffering_all_files_would_need_mib': round(args.concurrency * len(data) / 1024 ** 2, 1),
|
||||
}
|
||||
text = json.dumps(report, indent=2)
|
||||
if args.output:
|
||||
Path(args.output).write_text(text + '\n')
|
||||
print(text)
|
||||
check(report['uploads_failed'] == 0, 'some uploads failed')
|
||||
check(max(peaks.values()) < limit, 'peak memory reached the limit')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import io
|
||||
from PIL import Image
|
||||
from client import Client, arguments, check, run
|
||||
from stack import SAMPLE, upload, wait_status
|
||||
|
||||
EXPECTED = {'thumbnail': ('image/jpeg', 'JPEG'), 'crop': ('image/jpeg', 'JPEG'), 'optimized': ('image/webp', 'WEBP')}
|
||||
|
||||
|
||||
def encode(img, fmt, **kw):
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, fmt, **kw)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def halves(w, h):
|
||||
img = Image.new('RGB', (w, h), (0, 0, 255))
|
||||
img.paste((255, 0, 0), (0, 0, w // 2, h))
|
||||
return img
|
||||
|
||||
|
||||
def process(c, alice, bob, chat, data, content_type='image/png'):
|
||||
att = upload(c, alice, chat, data, content_type)['json']
|
||||
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60)
|
||||
check(ready['status'] == 'ready' and ready['error_code'] is None, 'attachment failed: %s' % ready['error_code'])
|
||||
check(sorted(v['name'] for v in ready['variants']) == sorted(EXPECTED), 'unexpected variant set')
|
||||
original = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes']
|
||||
check(original == data and ready['sha256'] == hashlib.sha256(data).hexdigest(), 'original changed')
|
||||
out = {}
|
||||
for v in ready['variants']:
|
||||
r = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + v['name'], actor=bob)
|
||||
body = r['bytes']
|
||||
check(r['headers'].get_content_type() == v['content_type'] == EXPECTED[v['name']][0], v['name'] + ' content type')
|
||||
check(len(body) == v['size_bytes'] and hashlib.sha256(body).hexdigest() == v['sha256'], v['name'] + ' checksum')
|
||||
img = Image.open(io.BytesIO(body))
|
||||
check(img.format == EXPECTED[v['name']][1] and img.size == (v['width'], v['height']), v['name'] + ' real size/format')
|
||||
check('exif' not in img.info, v['name'] + ' keeps EXIF')
|
||||
img.load()
|
||||
out[v['name']] = img.convert('RGB')
|
||||
return att, out
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Thumbnail, crop and optimized variant rules')
|
||||
c = Client(args.base_url, args.ca_file, timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
|
||||
att, v = process(c, alice, bob, chat, SAMPLE)
|
||||
check(v['thumbnail'].size == (256, 160) and v['crop'].size == (128, 128) and v['optimized'].size == (320, 200), 'fixture sizes')
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=bogus', 400, actor=bob)
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 404, actor=eve)
|
||||
|
||||
_, v = process(c, alice, bob, chat, encode(halves(2000, 1000), 'PNG'))
|
||||
check(v['thumbnail'].size == (256, 128) and v['optimized'].size == (1280, 640) and v['crop'].size == (128, 128), 'large image sizes')
|
||||
|
||||
_, v = process(c, alice, bob, chat, encode(halves(100, 50), 'PNG'))
|
||||
check(v['thumbnail'].size == (100, 50) and v['optimized'].size == (100, 50), 'small image must not be upscaled')
|
||||
check(v['crop'].size == (128, 128), 'crop must be 128x128')
|
||||
|
||||
_, v = process(c, alice, bob, chat, encode(Image.new('RGBA', (300, 300), (0, 0, 0, 0)), 'PNG'))
|
||||
for name, img in v.items():
|
||||
check(min(img.getpixel((img.width // 2, img.height // 2))) >= 250, name + ': alpha not composited on white')
|
||||
|
||||
exif = Image.Exif()
|
||||
exif[0x0112] = 6
|
||||
_, v = process(c, alice, bob, chat, encode(halves(300, 200), 'JPEG', quality=95, exif=exif.tobytes()), 'image/jpeg')
|
||||
t = v['thumbnail']
|
||||
check(t.size == (170, 256) and v['optimized'].size == (200, 300), 'EXIF orientation not applied to dimensions')
|
||||
top, bottom = t.getpixel((85, 20)), t.getpixel((85, 236))
|
||||
check(top[0] > 200 and top[2] < 60 and bottom[2] > 200 and bottom[0] < 60, 'EXIF rotation direction wrong')
|
||||
print('PASS: thumbnail/crop/optimized sizes, formats, checksums, no upscale, alpha on white, EXIF orientation, no EXIF in variants')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Executable
+84
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from client import Client, arguments, check, run
|
||||
from image_gen import png
|
||||
from stack import SAMPLE, compose, job_row, psql, restore_worker, upload, wait_status
|
||||
|
||||
LIMIT_MIB = 768
|
||||
|
||||
|
||||
def sample(stop, state):
|
||||
while not stop.is_set():
|
||||
out = compose('exec', '-T', 'worker', 'grep', 'VmRSS', '/proc/1/status', check_rc=False).stdout.split()
|
||||
if len(out) > 1:
|
||||
state['rss'] = max(state['rss'], int(out[1]) / 1024)
|
||||
n = psql("select count(*) from jobs where status = 'processing'")
|
||||
state['processing'] = max(state['processing'], int(n or 0))
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
def main():
|
||||
def configure(p):
|
||||
p.add_argument('--huge', type=int, default=6)
|
||||
p.add_argument('--small', type=int, default=20)
|
||||
p.add_argument('--output')
|
||||
args = arguments('Worker concurrency, memory and time limits', configure)
|
||||
c = Client(args.base_url, args.ca_file, timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
concurrency = int(compose('exec', '-T', 'worker', 'printenv', 'WORKER_CONCURRENCY').stdout.strip())
|
||||
huge = png(6320, 6320)
|
||||
|
||||
state, stop = {'rss': 0, 'processing': 0}, threading.Event()
|
||||
t = threading.Thread(target=sample, args=(stop, state))
|
||||
t.start()
|
||||
started = time.monotonic()
|
||||
ids = [upload(c, alice, chat, huge)['json']['id'] for _ in range(args.huge)]
|
||||
ids += [upload(c, alice, chat, SAMPLE)['json']['id'] for _ in range(args.small)]
|
||||
accepted = time.monotonic() - started
|
||||
results = [wait_status(c, bob, i, ('ready', 'failed'), 300)['status'] for i in ids]
|
||||
elapsed = time.monotonic() - started
|
||||
stop.set()
|
||||
t.join()
|
||||
container = compose('ps', '-q', 'worker').stdout.strip()
|
||||
oom = subprocess.run(['docker', 'inspect', '-f', '{{.RestartCount}} {{.State.OOMKilled}}', container],
|
||||
capture_output=True, text=True).stdout.split()
|
||||
|
||||
compose('stop', 'worker')
|
||||
slow = upload(c, alice, chat, huge)['json']
|
||||
compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'JOB_TIMEOUT_SECONDS': '1'})
|
||||
timed = wait_status(c, bob, slow['id'], ('ready', 'failed'), 120)
|
||||
_, attempts, last_error = job_row(slow['id'])
|
||||
restore_worker()
|
||||
|
||||
report = {
|
||||
'huge_images': {'count': args.huge, 'pixels': 6320 * 6320, 'bytes': len(huge)},
|
||||
'small_images': args.small,
|
||||
'worker_concurrency': concurrency,
|
||||
'max_processing_jobs_observed': state['processing'],
|
||||
'all_ready': results.count('ready') == len(results),
|
||||
'accept_seconds': round(accepted, 2),
|
||||
'all_ready_seconds': round(elapsed, 1),
|
||||
'worker_peak_rss_mib': round(state['rss'], 1),
|
||||
'worker_memory_limit_mib': LIMIT_MIB,
|
||||
'worker_restarts_oom': oom,
|
||||
'timeout_case': {'job_timeout_seconds': 1, 'status': timed['status'], 'error_code': timed['error_code'],
|
||||
'attempts': attempts, 'last_error': last_error},
|
||||
}
|
||||
text = json.dumps(report, indent=2)
|
||||
if args.output:
|
||||
Path(args.output).write_text(text + '\n')
|
||||
print(text)
|
||||
check(report['all_ready'], 'some jobs failed under load')
|
||||
check(state['processing'] <= concurrency, 'more jobs processing than WORKER_CONCURRENCY')
|
||||
check(state['rss'] < LIMIT_MIB and oom == ['0', 'false'], 'worker reached memory limit')
|
||||
check(timed['status'] == 'failed' and timed['error_code'] == 'retry_exhausted' and 'deadline' in last_error,
|
||||
'timeout must be a bounded transient failure')
|
||||
print('PASS: bounded worker concurrency, memory below limit, job timeout enforced')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,448 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math/rand/v2"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
var errSource = errors.New("staging object missing or corrupted")
|
||||
|
||||
type Worker struct {
|
||||
db *pgxpool.Pool
|
||||
store *Store
|
||||
broker *Broker
|
||||
log *slog.Logger
|
||||
owner string
|
||||
timeout time.Duration
|
||||
lease time.Duration
|
||||
backoffBase time.Duration
|
||||
backoffMax time.Duration
|
||||
maxBytes int64
|
||||
maxPixels int64
|
||||
fault string
|
||||
}
|
||||
|
||||
type claimedJob struct {
|
||||
id, attachmentID, owner string
|
||||
attempt int
|
||||
}
|
||||
|
||||
type source struct {
|
||||
stagingKey, contentType, sha256 string
|
||||
size int64
|
||||
}
|
||||
|
||||
func finalKey(attachmentID string) string {
|
||||
return objectPrefix + attachmentID + "/original"
|
||||
}
|
||||
|
||||
func variantKey(attachmentID string, spec variantSpec) string {
|
||||
return fmt.Sprintf("%s%s/v%d/%s.%s", objectPrefix, attachmentID, pipelineVersion, spec.name, spec.ext)
|
||||
}
|
||||
|
||||
func (w *Worker) inject(point string) {
|
||||
if w.fault == point {
|
||||
w.log.Error("fault injected", "point", point)
|
||||
os.Exit(3)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) backoff(attempt int) time.Duration {
|
||||
d := w.backoffBase << (attempt - 1)
|
||||
if d <= 0 || d > w.backoffMax {
|
||||
d = w.backoffMax
|
||||
}
|
||||
return d/2 + rand.N(d/2+1)
|
||||
}
|
||||
|
||||
func (w *Worker) claim(ctx context.Context, jobID string) (*claimedJob, error) {
|
||||
tx, err := w.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
job := &claimedJob{id: jobID, owner: w.owner}
|
||||
err = tx.QueryRow(ctx, `
|
||||
update jobs set status = 'processing', lease_owner = $2,
|
||||
lease_until = now() + make_interval(secs => $3), updated_at = now()
|
||||
where id = $1::uuid and status = 'queued' and next_attempt_at <= now()
|
||||
returning attachment_id::text, attempt`, jobID, w.owner, w.lease.Seconds()).Scan(&job.attachmentID, &job.attempt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `update attachments set status = 'processing', updated_at = now() where id = $1::uuid`, job.attachmentID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return job, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (w *Worker) fetch(ctx context.Context, src source) ([]byte, error) {
|
||||
obj, info, err := w.store.Get(ctx, src.stagingKey)
|
||||
if err != nil {
|
||||
if isNoSuchKey(err) {
|
||||
return nil, errSource
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer obj.Close()
|
||||
if info.Size != src.size || info.Size > w.maxBytes {
|
||||
return nil, errSource
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(obj, w.maxBytes+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
if int64(len(data)) != src.size || hex.EncodeToString(sum[:]) != src.sha256 {
|
||||
return nil, errSource
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (w *Worker) execute(ctx context.Context, job *claimedJob) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, w.timeout)
|
||||
defer cancel()
|
||||
var src source
|
||||
err := w.db.QueryRow(ctx, `
|
||||
select coalesce(staging_key, ''), content_type, sha256, size_bytes from attachments where id = $1::uuid`,
|
||||
job.attachmentID).Scan(&src.stagingKey, &src.contentType, &src.sha256, &src.size)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := w.fetch(ctx, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
variants, err := renderVariants(ctx, data, w.maxPixels)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
original := finalKey(job.attachmentID)
|
||||
if err := w.store.Put(ctx, original, bytes.NewReader(data), int64(len(data)), src.contentType); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, v := range variants {
|
||||
if err := w.store.Put(ctx, variantKey(job.attachmentID, v.spec), bytes.NewReader(v.data), int64(len(v.data)), v.spec.contentType); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
w.inject("after_store")
|
||||
|
||||
tx, err := w.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
tag, err := tx.Exec(ctx, `
|
||||
update jobs set status = 'ready', lease_owner = null, lease_until = null, last_error = null, updated_at = now()
|
||||
where id = $1::uuid and status = 'processing' and lease_owner = $2 and attempt = $3`, job.id, job.owner, job.attempt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
w.log.Warn("lease lost, result discarded", "job_id", job.id, "attempt", job.attempt)
|
||||
return nil
|
||||
}
|
||||
for _, v := range variants {
|
||||
_, err := tx.Exec(ctx, `
|
||||
insert into attachment_variants(attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256)
|
||||
values($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
on conflict (attachment_id, pipeline_version, name) do update
|
||||
set object_key = excluded.object_key, content_type = excluded.content_type, size_bytes = excluded.size_bytes,
|
||||
width = excluded.width, height = excluded.height, sha256 = excluded.sha256`,
|
||||
job.attachmentID, pipelineVersion, v.spec.name, variantKey(job.attachmentID, v.spec), v.spec.contentType,
|
||||
len(v.data), v.width, v.height, v.sha256)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
update attachments set status = 'ready', error_code = null, object_key = $2, updated_at = now()
|
||||
where id = $1::uuid`, job.attachmentID, original); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
w.log.Info("job ready", "job_id", job.id, "attachment_id", job.attachmentID, "attempt", job.attempt)
|
||||
w.inject("before_ack")
|
||||
if err := w.store.Remove(ctx, src.stagingKey); err != nil {
|
||||
w.log.Warn("staging cleanup deferred to janitor", "attachment_id", job.attachmentID, "err", err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) settle(ctx context.Context, job *claimedJob, permanent bool, code, reason string) error {
|
||||
tx, err := w.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
var maxAttempts int
|
||||
if err := tx.QueryRow(ctx, `select max_attempts from jobs where id = $1::uuid`, job.id).Scan(&maxAttempts); err != nil {
|
||||
return err
|
||||
}
|
||||
fence := ` where id = $1::uuid and status = 'processing' and lease_owner = $2 and attempt = $3`
|
||||
if !permanent && job.attempt < maxAttempts {
|
||||
delay := w.backoff(job.attempt)
|
||||
tag, err := tx.Exec(ctx, `
|
||||
update jobs set status = 'queued', attempt = attempt + 1, next_attempt_at = now() + make_interval(secs => $4),
|
||||
lease_owner = null, lease_until = null, last_error = $5, updated_at = now()`+fence,
|
||||
job.id, job.owner, job.attempt, delay.Seconds(), reason)
|
||||
if err != nil || tag.RowsAffected() == 0 {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
insert into outbox(job_id, attempt, available_at, created_at)
|
||||
values($1::uuid, $2, now() + make_interval(secs => $3), now())`, job.id, job.attempt+1, delay.Seconds()); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `update attachments set status = 'queued', updated_at = now() where id = $1::uuid`, job.attachmentID); err != nil {
|
||||
return err
|
||||
}
|
||||
w.log.Warn("job retry scheduled", "job_id", job.id, "attempt", job.attempt, "next_attempt", job.attempt+1,
|
||||
"delay_ms", delay.Milliseconds(), "reason", reason)
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
if !permanent {
|
||||
code = "retry_exhausted"
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `
|
||||
update jobs set status = 'failed', lease_owner = null, lease_until = null, last_error = $4, updated_at = now()`+fence,
|
||||
job.id, job.owner, job.attempt, reason)
|
||||
if err != nil || tag.RowsAffected() == 0 {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
update attachments set status = 'failed', error_code = $2, updated_at = now() where id = $1::uuid`, job.attachmentID, code); err != nil {
|
||||
return err
|
||||
}
|
||||
w.log.Error("job failed", "job_id", job.id, "attempt", job.attempt, "error_code", code, "reason", reason)
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (w *Worker) handle(ctx context.Context, msg redis.XMessage) bool {
|
||||
var m jobMessage
|
||||
raw, _ := msg.Values["job"].(string)
|
||||
if err := json.Unmarshal([]byte(raw), &m); err != nil || m.SchemaVersion != 1 || m.Type != jobType || !isUUID(m.JobID) {
|
||||
w.log.Warn("invalid job message dropped", "stream_id", msg.ID)
|
||||
return true
|
||||
}
|
||||
job, err := w.claim(ctx, m.JobID)
|
||||
if err != nil {
|
||||
w.log.Warn("claim failed", "job_id", m.JobID, "err", err.Error())
|
||||
return false
|
||||
}
|
||||
if job == nil {
|
||||
var status string
|
||||
var attempt int
|
||||
_ = w.db.QueryRow(ctx, `select status, attempt from jobs where id = $1::uuid`, m.JobID).Scan(&status, &attempt)
|
||||
w.log.Info("job delivery skipped", "job_id", m.JobID, "message_attempt", m.Attempt, "status", status, "attempt", attempt)
|
||||
return true
|
||||
}
|
||||
err = w.execute(ctx, job)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
permanent, code := false, "processing_failed"
|
||||
switch {
|
||||
case errors.Is(err, errInvalidImage):
|
||||
permanent, code = true, "invalid_image"
|
||||
case errors.Is(err, errSource):
|
||||
permanent = true
|
||||
}
|
||||
if err := w.settle(ctx, job, permanent, code, err.Error()); err != nil {
|
||||
w.log.Warn("settle failed", "job_id", job.id, "err", err.Error())
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (w *Worker) read(ctx context.Context, consumer, id string, block time.Duration) ([]redis.XMessage, error) {
|
||||
res, err := w.broker.rdb.XReadGroup(ctx, &redis.XReadGroupArgs{
|
||||
Group: jobGroup, Consumer: consumer, Streams: []string{jobStream, id}, Count: 1, Block: block,
|
||||
}).Result()
|
||||
if len(res) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
return res[0].Messages, err
|
||||
}
|
||||
|
||||
func (w *Worker) consume(ctx context.Context, consumer string) {
|
||||
pending := true
|
||||
nextClaim := time.Now().Add(w.lease)
|
||||
for ctx.Err() == nil {
|
||||
var msgs []redis.XMessage
|
||||
var err error
|
||||
switch {
|
||||
case pending:
|
||||
msgs, err = w.read(ctx, consumer, "0", -1)
|
||||
pending = len(msgs) > 0
|
||||
case time.Now().After(nextClaim):
|
||||
msgs, _, err = w.broker.rdb.XAutoClaim(ctx, &redis.XAutoClaimArgs{
|
||||
Stream: jobStream, Group: jobGroup, Consumer: consumer, MinIdle: w.lease, Start: "0-0", Count: 1,
|
||||
}).Result()
|
||||
nextClaim = time.Now().Add(15 * time.Second)
|
||||
default:
|
||||
msgs, err = w.read(ctx, consumer, ">", 5*time.Second)
|
||||
}
|
||||
if err != nil && !errors.Is(err, redis.Nil) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if strings.Contains(err.Error(), "NOGROUP") {
|
||||
_ = w.broker.ensureGroup(ctx)
|
||||
}
|
||||
w.log.Warn("broker read failed", "consumer", consumer, "err", err.Error())
|
||||
time.Sleep(2 * time.Second)
|
||||
continue
|
||||
}
|
||||
for _, msg := range msgs {
|
||||
if !w.handle(context.WithoutCancel(ctx), msg) {
|
||||
pending = false
|
||||
continue
|
||||
}
|
||||
if err := w.broker.rdb.XAck(context.WithoutCancel(ctx), jobStream, jobGroup, msg.ID).Err(); err != nil {
|
||||
w.log.Warn("ack failed", "stream_id", msg.ID, "err", err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) reap(ctx context.Context) error {
|
||||
rows, err := w.db.Query(ctx, `
|
||||
select id::text, attachment_id::text, attempt, coalesce(lease_owner, '')
|
||||
from jobs where status = 'processing' and lease_until < now() limit 50`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
jobs, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (*claimedJob, error) {
|
||||
j := &claimedJob{}
|
||||
return j, row.Scan(&j.id, &j.attachmentID, &j.attempt, &j.owner)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, j := range jobs {
|
||||
w.log.Warn("lease expired", "job_id", j.id, "attempt", j.attempt)
|
||||
if err := w.settle(ctx, j, false, "", "lease expired"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runWorker(db *pgxpool.Pool, store *Store, broker *Broker, log *slog.Logger, w *Worker, concurrency int) {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
|
||||
defer stop()
|
||||
for {
|
||||
err := broker.ensureGroup(ctx)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
log.Warn("broker not ready", "err", err.Error())
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
host, _ := os.Hostname()
|
||||
var wg sync.WaitGroup
|
||||
wg.Go(func() {
|
||||
for ctx.Err() == nil {
|
||||
if err := w.reap(ctx); err != nil && ctx.Err() == nil {
|
||||
log.Warn("lease reaper failed", "err", err.Error())
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(5 * time.Second):
|
||||
}
|
||||
}
|
||||
})
|
||||
for i := range concurrency {
|
||||
consumer := fmt.Sprintf("%s-%d", host, i)
|
||||
wg.Go(func() { w.consume(ctx, consumer) })
|
||||
}
|
||||
log.Info("worker started", "owner", w.owner, "concurrency", concurrency, "lease_s", w.lease.Seconds())
|
||||
<-ctx.Done()
|
||||
log.Info("worker stopping")
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func replayJob(ctx context.Context, db *pgxpool.Pool, attachmentID string, maxAttempts int) (string, error) {
|
||||
tx, err := db.Begin(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
var jobID, status string
|
||||
var attempt int
|
||||
var lastErr *string
|
||||
err = tx.QueryRow(ctx, `select id::text, status, attempt, last_error from jobs where attachment_id = $1::uuid for update`,
|
||||
attachmentID).Scan(&jobID, &status, &attempt, &lastErr)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "not_found", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if status != "failed" {
|
||||
return "skipped_" + status, nil
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `insert into job_replays(job_id, previous_error, replayed_at) values($1::uuid, $2, now())`, jobID, lastErr); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
update jobs set status = 'queued', attempt = attempt + 1, max_attempts = attempt + $2, next_attempt_at = now(),
|
||||
last_error = null, updated_at = now()
|
||||
where id = $1::uuid`, jobID, maxAttempts); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `insert into outbox(job_id, attempt, available_at, created_at) values($1::uuid, $2, now(), now())`, jobID, attempt+1); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `update attachments set status = 'queued', error_code = null, updated_at = now() where id = $1::uuid`, attachmentID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "queued", tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func listFailed(ctx context.Context, db *pgxpool.Pool) ([]map[string]any, error) {
|
||||
rows, err := db.Query(ctx, `
|
||||
select j.attachment_id::text, j.id::text, j.attempt, coalesce(a.error_code, ''), coalesce(j.last_error, ''), j.updated_at
|
||||
from jobs j join attachments a on a.id = j.attachment_id
|
||||
where j.status = 'failed' order by j.updated_at`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pgx.CollectRows(rows, func(row pgx.CollectableRow) (map[string]any, error) {
|
||||
var att, job, code, reason string
|
||||
var attempt int
|
||||
var updated time.Time
|
||||
err := row.Scan(&att, &job, &attempt, &code, &reason, &updated)
|
||||
return map[string]any{"attachment_id": att, "job_id": job, "attempt": attempt, "error_code": code,
|
||||
"last_error": reason, "failed_at": formatTime(updated)}, err
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user