From 4adc6c58c8806ebdf008bab91654c43eaa9b7eb3 Mon Sep 17 00:00:00 2001 From: gera Date: Thu, 19 Nov 2026 23:10:55 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D0=BB=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B1=D0=BB=D1=8E=D0=B4=D0=B0=D0=B5=D0=BC=D0=BE?= =?UTF-8?q?=D1=81=D1=82=D1=8C:=20OpenTelemetry,=20Prometheus,=20Loki,=20Te?= =?UTF-8?q?mpo,=20Grafana?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 4 + CHANGES.md | 37 + Dockerfile | 19 + REPORT.md | 225 ++- attachments.go | 534 +++++++ auth.go | 131 ++ broker.go | 162 ++ compose.yaml | 475 ++++++ db.go | 59 + evidence/attachments.txt | 1 + evidence/dashboard.txt | 2 + evidence/failover-1.txt | 4 + evidence/failover-2.txt | 4 + evidence/failover-each.json | 71 + evidence/faults.txt | 7 + evidence/idempotency.txt | 1 + evidence/incident-database.txt | 4 + evidence/incident-storage.txt | 5 + evidence/incident-worker.txt | 4 + evidence/load.json | 17 + evidence/observability.txt | 5 + evidence/persistence.txt | 32 + evidence/queue-recovery.txt | 5 + evidence/rolling.json | 24 + evidence/s3-checks.txt | 1 + evidence/session-revocation.txt | 1 + evidence/smoke.txt | 4 + evidence/telemetry-persistence.txt | 2 + evidence/upload-memory.json | 18 + evidence/variants.txt | 1 + evidence/worker-load.json | 26 + go.mod | 61 + go.sum | 142 ++ handlers.go | 1095 ++++++++++++++ imaging.go | 176 +++ infra/grafana/dashboards/messenger.json | 1317 +++++++++++++++++ .../provisioning/dashboards/dashboards.yaml | 9 + .../provisioning/datasources/datasources.yaml | 34 + infra/loki/loki.yaml | 47 + infra/otel/collector.yaml | 77 + infra/prometheus/prometheus.yml | 19 + infra/prometheus/rules/alerts.yml | 23 + infra/runbook.md | 31 + infra/tempo/tempo.yaml | 27 + lb/haproxy.cfg | 38 + main.go | 214 +++ migrate.go | 84 ++ migrations/0001_init.sql | 38 + migrations/0002_idempotency.sql | 10 + migrations/0003_attachments.sql | 22 + migrations/0004_jobs.sql | 60 + migrations/0005_trace_context.sql | 5 + models.go | 33 + s3.go | 117 ++ scripts/mksecrets.sh | 29 + scripts/restart.sh | 7 + scripts/rolling-update.sh | 16 + scripts/stop-one.sh | 10 + telemetry.go | 352 +++++ tests/attachments.py | 89 ++ tests/dashboard.py | 48 + tests/faults.py | 157 ++ tests/idempotency.py | 42 + tests/image_gen.py | 23 + tests/incidents.py | 187 +++ tests/obs.py | 125 ++ tests/observability.py | 128 ++ tests/queue_recovery.py | 68 + tests/s3_checks.py | 137 ++ tests/scenario.py | 155 ++ tests/session_revocation.py | 31 + tests/stack.py | 92 ++ tests/telemetry_persistence.py | 32 + tests/upload_memory.py | 106 ++ tests/variants.py | 76 + tests/worker_load.py | 84 ++ worker.go | 489 ++++++ 77 files changed, 8031 insertions(+), 16 deletions(-) create mode 100644 CHANGES.md create mode 100644 Dockerfile create mode 100644 attachments.go create mode 100644 auth.go create mode 100644 broker.go create mode 100644 compose.yaml create mode 100644 db.go create mode 100644 evidence/attachments.txt create mode 100644 evidence/dashboard.txt create mode 100644 evidence/failover-1.txt create mode 100644 evidence/failover-2.txt create mode 100644 evidence/failover-each.json create mode 100644 evidence/faults.txt create mode 100644 evidence/idempotency.txt create mode 100644 evidence/incident-database.txt create mode 100644 evidence/incident-storage.txt create mode 100644 evidence/incident-worker.txt create mode 100644 evidence/load.json create mode 100644 evidence/observability.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/queue-recovery.txt create mode 100644 evidence/rolling.json create mode 100644 evidence/s3-checks.txt create mode 100644 evidence/session-revocation.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/telemetry-persistence.txt create mode 100644 evidence/upload-memory.json create mode 100644 evidence/variants.txt create mode 100644 evidence/worker-load.json create mode 100644 go.mod create mode 100644 go.sum create mode 100644 handlers.go create mode 100644 imaging.go create mode 100644 infra/grafana/dashboards/messenger.json create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 infra/grafana/provisioning/datasources/datasources.yaml create mode 100644 infra/loki/loki.yaml create mode 100644 infra/otel/collector.yaml create mode 100644 infra/prometheus/prometheus.yml create mode 100644 infra/prometheus/rules/alerts.yml create mode 100644 infra/runbook.md create mode 100644 infra/tempo/tempo.yaml create mode 100644 lb/haproxy.cfg create mode 100644 main.go create mode 100644 migrate.go create mode 100644 migrations/0001_init.sql create mode 100644 migrations/0002_idempotency.sql create mode 100644 migrations/0003_attachments.sql create mode 100644 migrations/0004_jobs.sql create mode 100644 migrations/0005_trace_context.sql create mode 100644 models.go create mode 100644 s3.go create mode 100755 scripts/mksecrets.sh create mode 100755 scripts/restart.sh create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/stop-one.sh create mode 100644 telemetry.go create mode 100644 tests/attachments.py create mode 100644 tests/dashboard.py create mode 100755 tests/faults.py create mode 100644 tests/idempotency.py create mode 100644 tests/image_gen.py create mode 100644 tests/incidents.py create mode 100644 tests/obs.py create mode 100644 tests/observability.py create mode 100755 tests/queue_recovery.py create mode 100644 tests/s3_checks.py create mode 100644 tests/scenario.py create mode 100644 tests/session_revocation.py create mode 100644 tests/stack.py create mode 100644 tests/telemetry_persistence.py create mode 100644 tests/upload_memory.py create mode 100755 tests/variants.py create mode 100755 tests/worker_load.py create mode 100644 worker.go diff --git a/.env.example b/.env.example index 2b32655..1739116 100644 --- a/.env.example +++ b/.env.example @@ -28,3 +28,7 @@ OTEL_SERVICE_NAME=messenger-api OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +CLEANUP_GRACE_SECONDS=600 +CLEANUP_INTERVAL_SECONDS=300 +JOB_BACKOFF_BASE_MS=1000 +JOB_BACKOFF_MAX_MS=30000 diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..49405e7 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,37 @@ +# Изменения lab6 относительно lab5 + +Бизнес-API не менялся. Контракт OpenAPI, схема `jobs.schema.json` и публичные тесты остались как в skeleton. + +## Приложение + +- `telemetry.go`: + - OTLP/HTTP экспорт трасс, метрик и логов в Collector с ограниченными очередями и повторами; + - `slog`-обработчик пишет JSON в stdout и тот же лог в OTLP; в stdout добавляются `service.name`, `event`, `trace_id`, `span_id`; + - инструменты метрик, process CPU/RSS, DB pool, очередь, saturation worker; + - трассировщик запросов pgx и вспомогательные функции spans и W3C trace context. +- `handlers.go`: middleware создаёт server span, извлекает входящий `traceparent`, считает `http.server.requests` и `http.server.request.duration` с route template и классом статуса. Health-запросы не создают spans и метрики. +- `broker.go`: сообщение получает поля `traceparent` и `tracestate` (разрешены `jobs.schema.json`); relay создаёт producer span `attachments.jobs publish` как потомка запроса загрузки. +- `worker.go`: consumer span `attachments.jobs process` продолжает trace из сообщения; метрики `jobs.processed`, `job.processing.duration`, `job.time_to_ready`, `worker.busy`; логи привязаны к trace. +- `attachments.go`: контекст trace сохраняется в `jobs.traceparent` и `jobs.tracestate` в той же транзакции, что и задача. Повторы, reaper и replay остаются в исходном trace. +- `s3.go`: spans `s3 put`, `s3 get`, `s3 remove` внутри существующих trace. +- `db.go`: трассировка запросов и метрики пула. +- `main.go`: телеметрия включается только для `api` и `worker`; метрики очереди регистрирует API. +- `migrations/0005_trace_context.sql`: колонки `traceparent`, `tracestate` в `jobs`, индекс очереди. +- Зависимости: `go.opentelemetry.io/otel` и OTLP/HTTP экспортёры, мост `otelslog`. + +## Инфраструктура + +- `compose.yaml`: сервисы `otel-collector`, `prometheus`, `loki`, `tempo`, `grafana` с закреплёнными digest, лимитами, healthcheck и volumes; сеть `telemetry`; образ приложения `mtusi-messenger:lab6`; OTEL-переменные в `api` и `worker`. +- `infra/otel/collector.yaml`: receivers OTLP, `memory_limiter`, `batch`, ограниченные очереди и повторы экспортёров, три pipeline. +- `infra/prometheus/`: конфигурация с OTLP receiver и alert rules `ApiHighErrorRate`, `QueueLagGrowing`. +- `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml`: хранилища логов и трасс на локальных volumes, retention 7 суток. +- `infra/grafana/`: provisioning datasources (с переходом лог → trace и trace → лог) и dashboard `messenger.json`. +- `infra/runbook.md`: runbook для alerts. +- `scripts/mksecrets.sh`: пароль Grafana. `scripts/restart.sh`: перезапускает хранилища телеметрии и Collector. +- `.env.example`: переменные очистки и backoff, которые читает compose. + +## Тесты + +Новые: `tests/obs.py` (запросы к Prometheus, Loki, Tempo), `tests/observability.py` (сигналы, цепочка trace, корреляция логов, retry в одном trace, отсутствие секретов), `tests/incidents.py` (остановка worker, блокировка БД, остановка хранилищ телеметрии), `tests/dashboard.py` (provisioning и запросы dashboard через Grafana). + +Перенесены из lab5 без изменений: `attachments.py`, `s3_checks.py`, `idempotency.py`, `session_revocation.py`, `variants.py`, `queue_recovery.py`, `faults.py`, `worker_load.py`, `upload_memory.py`, `scenario.py`, `stack.py`, `image_gen.py`. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..94f0316 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM golang:1.26-alpine@sha256:c95332c2af86b6d89b91bd0500f4b9529ccbd090a0d1855c6d1ceaa142ae8615 AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY *.go ./ +COPY migrations ./migrations +RUN CGO_ENABLED=0 GOOS=linux go build -tags nodynamic -trimpath -ldflags="-s -w" -o /out/messenger . + +FROM alpine:3.21@sha256:ce64758a109eb420d874a118f87920e625e12d3634e03b4a5573fd9f6e5d3507 +RUN adduser -D -u 10001 app +WORKDIR /app +COPY --from=build /out/messenger /app/messenger +USER 10001 +EXPOSE 8080 +ENV APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +STOPSIGNAL SIGTERM +HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \ + CMD wget -qO- http://127.0.0.1:8080/health/ready >/dev/null || exit 1 +ENTRYPOINT ["/app/messenger"] diff --git a/REPORT.md b/REPORT.md index b97ae42..184c445 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,230 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Березин Герман / БВТ2304 +- Целевая оценка: 4 +- Commit/PR: после публикации +- Использованные источники, библиотеки и помощь ИИ: Go 1.26, OpenTelemetry Go SDK 1.47 (`otel`, `sdk`, `sdk/metric`, `sdk/log`, экспортёры OTLP/HTTP, мост `otelslog` 0.21), pgx, go-redis, minio-go; OpenTelemetry Collector contrib 0.116.1, Prometheus 3.1.0, Loki 3.3.2, Tempo 2.7.0, Grafana 11.3.0; документация OTel Collector configuration, OTel propagation, W3C Trace Context, Prometheus histograms, Grafana provisioning из README; код lab5 перенесён за основу; ИИ-ассистент использован при разработке ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker 29 с Compose v2 и Python 3.9+. Для `tests/variants.py` нужен Pillow. + +```sh +cp .env.example .env +sh scripts/mksecrets.sh +docker compose --env-file .env up -d --build --wait +make check && make test +``` + +`mksecrets.sh` создаёт секреты в `./secrets` (вне Git), в том числе пароль Grafana: `cat secrets/grafana_admin_password`, пользователь `admin`. Существующие файлы не перезаписываются. + +Порядок старта: postgres, valkey, broker, s3, prometheus, loki, tempo → `migrate` → `s3-init`, otel-collector, grafana → `api` × 2, `worker`, `janitor` → `lb`. Публичный адрес API: `http://localhost:8080`. Интерфейсы на `127.0.0.1`: Grafana `:3000`, Prometheus `:9090` (alerts: `/alerts`), Loki `:3100`, Tempo `:3200`. Dashboard и datasources подключаются автоматически из `infra/grafana/`, вручную ничего набирать не нужно. После первого старта S3 принимает запись через 10–30 с. + +Остановка без потери данных: `docker compose --env-file .env stop`. Намеренное удаление учебных данных: `docker compose --env-file .env down -v`. + +Проверки (стенд запущен): + +```sh +python3 tests/dashboard.py +python3 tests/observability.py +python3 tests/incidents.py --only worker +python3 tests/incidents.py --only database +python3 tests/incidents.py --only storage +python3 tests/telemetry_persistence.py +python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/scenario.py --mode failover-each --output evidence/failover-each.json +python3 tests/scenario.py --mode rolling --output evidence/rolling.json +python3 tests/variants.py +python3 tests/queue_recovery.py +python3 tests/faults.py +python3 tests/worker_load.py --output evidence/worker-load.json +python3 tests/attachments.py +python3 tests/s3_checks.py +python3 tests/upload_memory.py --output evidence/upload-memory.json +python3 tests/idempotency.py +python3 tests/session_revocation.py +``` + +`incidents.py`, `observability.py`, `faults.py`, `queue_recovery.py`, `worker_load.py` сами останавливают и пересоздают сервисы и возвращают их. После `make test-failover` реплику возвращают командой `docker compose --env-file .env up -d --wait api`. Операторские команды worker и разбор alerts: `infra/runbook.md`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +клиент → HAProxy → api ×2 ──OTLP──┐ + │ trace context в jobs.traceparent → outbox → сообщение broker + ▼ │ + PostgreSQL / S3 / broker ▼ + worker ──OTLP──┤ + ▼ + otel-collector (memory_limiter, batch, очереди) + метрики ─→ Prometheus логи ─→ Loki трассы ─→ Tempo + ▼ + Grafana (datasources и dashboard из Git) +``` + +Бизнес-API, PostgreSQL, S3, broker, outbox, worker и retry перенесены из lab5 без изменения смысла (подробности в `CHANGES.md`). Изменено: добавлена телеметрия, миграция `0005_trace_context.sql`, сервисы наблюдаемости. + +| Роль | Сервис | Образ | Готовность | Volume | Сеть | Назначение | +| --- | --- | --- | --- | --- | --- | --- | +| collector | `otel-collector` | otel/opentelemetry-collector-contrib:0.116.1@sha256:d0ebf652… | запуск после готовности хранилищ | нет | backend, telemetry | принимает OTLP, ограничивает память, батчит, раздаёт сигналы | +| метрики | `prometheus` | prom/prometheus:v3.1.0@sha256:6559acbd… | `/-/ready` | prometheus_data | telemetry, 127.0.0.1:9090 | хранит метрики (OTLP receiver), считает alert rules | +| логи | `loki` | grafana/loki:3.3.2@sha256:8af2de1a… | `/ready` | loki_data | telemetry, 127.0.0.1:3100 | хранит структурные логи, ищет по trace_id | +| трассы | `tempo` | grafana/tempo:2.7.0@sha256:12e904cc… | `/ready` | tempo_data | telemetry, 127.0.0.1:3200 | хранит трассы, поиск TraceQL | +| интерфейс | `grafana` | grafana/grafana:11.3.0@sha256:a0f88123… | `/api/health` | grafana_data | telemetry, 127.0.0.1:3000 | dashboard и запросы к трём источникам, сам данные не хранит | +| api ×2, worker, postgres, valkey, broker, s3, lb | как в lab5 | образ приложения `mtusi-messenger:lab6` | как в lab5 | как в lab5 | backend, storage | см. REPORT lab5 | + +Bootstrap: `migrate` применяет `migrations/*.sql`; provisioning Grafana читается при старте из `infra/grafana/provisioning`; Prometheus, Loki и Tempo стартуют из конфигов в `infra/`. Повторный запуск `up -d` безопасен. Пароль Grafana передаётся файлом secret (`GF_SECURITY_ADMIN_PASSWORD__FILE`). + +- **Collector** (`infra/otel/collector.yaml`): receivers OTLP gRPC/HTTP; processors `memory_limiter` (проверка 1 с, лимит 160 MiB, spike 40 MiB; контейнер 256 MiB) и `batch`; exporters `otlphttp` в Prometheus (`/api/v1/otlp`) и Loki (`/otlp`), `otlp` gRPC в Tempo. У каждого экспортёра очередь `sending_queue` на 100 батчей и повтор до 20 с (`retry_on_failure`), после чего данные отбрасываются. Collector не хранит данные на диске. +- **Приложение.** SDK экспортирует по OTLP/HTTP: батч трасс (очередь 2048 spans, таймаут 5 с), метрики раз в 10 с, батч логов (очередь 2048, интервал 1 с); повтор каждого экспорта не дольше 8 с. Недоступный Collector не блокирует запросы: полная очередь отбрасывает записи, ошибки экспорта считает `telemetry.export.errors`. +- **Распространение контекста.** Входящий `traceparent` запроса извлекается. Upload сохраняет контекст server span в `jobs.traceparent`/`tracestate` в той же транзакции, что и задачу. Relay берёт его из строки задачи, создаёт producer span `attachments.jobs publish` (родитель — запрос) и кладёт новый `traceparent` в сообщение (поля разрешены `jobs.schema.json`). Worker извлекает контекст и создаёт consumer span `attachments.jobs process` с родителем producer span. Повторы, reaper и `replay` публикуют сообщение из той же строки, поэтому все попытки лежат в одном trace, у каждой свой producer и consumer span. +- **Spans:** server span на запрос (имя `METHOD route template`), `db <операция>` на каждый запрос pgx (текст SQL без параметров), `s3 put/get/remove`, `image render`. Spans создаются только внутри существующего trace: фоновые опросы (relay, reaper, чтение stream) не порождают отдельных trace. +- **Логи.** JSON в stdout и тот же лог по OTLP в Loki. Поля: время, уровень, `service.name`, `event`, `trace_id`, `span_id` при активном span, технические поля события. Resource attributes: `service.name` (`messenger-api` / `messenger-worker`), `service.version`, `service.instance.id` (id контейнера), `deployment.environment.name=lab`. +- **Что не попадает в телеметрию.** Заголовок Authorization, токены, пароли, тела сообщений, содержимое изображений, URL с параметрами (в spans только `http.route` — шаблон), значения параметров SQL. В labels метрик нет user_id, chat_id, message_id, attachment_id, request_id, filename, raw URL. В логах и spans есть технические `request_id`, `job_id`, `attachment_id` (синтетические учебные данные): по ним находится нужный trace. +- **Хранилища и сохранность.** Prometheus, Loki и Tempo хранят данные 7 суток на своих volumes (Loki с WAL и сбросом при остановке). Историю после перезапуска проверяет `tests/telemetry_persistence.py`. + +### Соответствие .env.example + +| Переменная | Где используется | +| --- | --- | +| `OTEL_SERVICE_NAME` | api: `messenger-api`; worker: `messenger-worker` (задаётся в compose) | +| `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_PROTOCOL` | compose, api и worker; SDK читает стандартные переменные | +| `OTEL_RESOURCE_ATTRIBUTES` | `deployment.environment.name`, SDK читает стандартную переменную | +| остальные | как в lab5 (`API_REPLICAS`, `S3_*`, `WORKER_CONCURRENCY`, `JOB_*`, `CLEANUP_*`, `STAGING_RETENTION_SECONDS`, `*_FILE`) | + +### Сигналы и соответствие контракту + +Имена OTel и их вид в Prometheus (единицы в имени добавляет трансляция OTLP → Prometheus). Для каждого сигнала один инструмент, дублей нет. + +| Контракт | Инструмент OTel (единицы) | В Prometheus | Измерения | +| --- | --- | --- | --- | +| HTTP requests | `http.server.requests` (counter) | `http_server_requests_total` | method, route template, класс статуса (`2xx`…) | +| HTTP duration | `http.server.request.duration` (histogram, с) | `http_server_request_duration_seconds_bucket` | method, route, класс статуса; buckets 0.005–10 с | +| Jobs processed / failed | `jobs.processed` (counter) | `jobs_processed_total` | job.type, outcome: `ready`, `retry`, `failed`, `skipped`, `lease_lost` | +| Job duration | `job.processing.duration` (histogram, с) | `job_processing_duration_seconds_bucket` | job.type, outcome | +| Time to ready | `job.time_to_ready` (histogram, с) | `job_time_to_ready_seconds_bucket` | от durable acceptance (`jobs.created_at`) до COMMIT ready | +| Queue depth / oldest age | `queue.depth` ({job}), `queue.oldest_job_age` (с) | `queue_depth`, `queue_oldest_job_age_seconds` | queue; из PostgreSQL (`jobs` в статусе `queued`), возраст от `next_attempt_at` | +| DB pool | `db.pool.connections`, `db.pool.max_connections`, `db.pool.empty_acquires`, `db.pool.acquire.wait` (с) | `db_pool_connections{state}`, `db_pool_max_connections`, `db_pool_empty_acquires_total`, `db_pool_acquire_wait_seconds_total` | state `used`/`idle` | +| Worker saturation | `worker.busy`, `worker.capacity` | `worker_busy`, `worker_capacity` | | +| CPU / RAM | `process.cpu.time` (с), `process.memory.usage` (байт) | `process_cpu_time_seconds_total`, `process_memory_usage_bytes` | процесс (api, worker) | +| Потери телеметрии | `telemetry.export.errors`; метрики Collector `otelcol_exporter_*` (scrape Prometheus) | `telemetry_export_errors_total`, `otelcol_exporter_send_failed_*`, `otelcol_exporter_queue_size` | | +| Логи, трассы | OTLP logs, OTLP traces | Loki (`service_name`, structured metadata), Tempo | `trace_id`, `span_id` | + +Метрики очереди публикует каждая API-реплика (одно значение из PostgreSQL), на dashboard и в alert применяется `max`, чтобы данные не пропадали при остановке worker. + +### SLI и SLO + +| Цель | SLI | SLO | +| --- | --- | --- | +| Доступность API | доля бизнес-запросов без ответа 5xx: `1 − sum(rate(http_server_requests_total{status_class="5xx"})) / sum(rate(http_server_requests_total))`; health-запросы не учитываются, 4xx (ошибки клиента, например неверный пароль) не считаются отказом | 99% за скользящие 30 суток | +| Время до ready | доля вложений с `job_time_to_ready_seconds ≤ 10`: `increase(…_bucket{le="10"}) / increase(…_count)` | 95% за 30 суток | + +Alert rules (`infra/prometheus/rules/alerts.yml`, runbook `infra/runbook.md`) — быстрые проверки расхода бюджета на коротких окнах, чтобы воспроизводить их на стенде: + +| Alert | Условие | Окно / `for` | Что означает | +| --- | --- | --- | --- | +| `ApiHighErrorRate` | доля 5xx > 5% | 1 мин / 20 с | бюджет доступности расходуется быстрее допустимого | +| `QueueLagGrowing` | `max(queue_oldest_job_age_seconds) > 20` | — / 20 с | worker не успевает или остановлен, SLO времени до ready под угрозой | + +Окно 30 суток на стенде не накапливается: показаны SLI за выбранный интервал dashboard и alerts, проверенные на коротком окне. Burn-rate с несколькими окнами не реализован (это трек 5). + +### Dashboard «Messenger observability» + +Файл `infra/grafana/dashboards/messenger.json`, импортируется provisioning'ом. Все 28 панелей и 34 запроса выполняются без ошибок (`tests/dashboard.py`). Ключевые панели: + +| Панель | Запрос | Единица | +| --- | --- | --- | +| RPS по маршрутам | `sum by (http_route) (rate(http_server_requests_total[$__rate_interval]))` | запросов/с | +| Доля ошибок | `(sum(rate(…{http_response_status_class="5xx"}[$__rate_interval])) or vector(0)) / sum(rate(http_server_requests_total[$__rate_interval]))` | доля (0–1) | +| Latency p50/p95/p99 | `histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket[$__rate_interval])))` | с | +| Jobs по outcome | `sum by (outcome) (rate(jobs_processed_total[$__rate_interval]))` | задач/с | +| Job duration p50/p95/p99 | `histogram_quantile(0.95, sum by (le) (rate(job_processing_duration_seconds_bucket[…])))` | с | +| Time to ready p50/p95/p99 | `histogram_quantile(…, sum by (le) (rate(job_time_to_ready_seconds_bucket[…])))` | с | +| Queue depth | `max by (queue) (queue_depth)` | задач | +| Oldest queued job age | `max by (queue) (queue_oldest_job_age_seconds)` | с | +| Worker saturation | `sum(worker_busy)`, `sum(worker_capacity)` | слотов | +| DB pool | `sum by (instance) (db_pool_connections{state="used",job="messenger-api"})`, `max(db_pool_max_connections)` | соединений | +| DB acquire wait | `sum by (job) (rate(db_pool_acquire_wait_seconds_total[…]))`, `rate(db_pool_empty_acquires_total[…])` | с/с, ожиданий/с | +| CPU, RSS | `rate(process_cpu_time_seconds_total[…])`, `process_memory_usage_bytes` | ядра, байты | +| SLI доступности, SLI ready ≤ 10 с | см. таблицу SLI | доля | +| Collector: очередь, потери, память | `otelcol_exporter_queue_size`, `rate(otelcol_exporter_send_failed_*[…])`, `otelcol_process_memory_rss` | батчей, записей/с, байты | +| Логи warn/error, трассы с ошибкой | Loki `{service_name=~"messenger-.*"} \| detected_level=~"warn\|error"`, TraceQL `{ status = error }` | | + +Percentiles берутся из histogram buckets суммой по всем репликам (`sum by (le)`), средние p95 реплик не используются. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke при включённой телеметрии | `make check && make test` | PASS | evidence/smoke.txt | +| Минимум lab5 сохранён | `tests/attachments.py`, `s3_checks.py`, `idempotency.py`, `session_revocation.py`, `variants.py`, `queue_recovery.py`, `faults.py`, `worker_load.py`, `upload_memory.py` | PASS; `s3_checks.py` допускает публикацию портов только на 127.0.0.1 и только у lb, Grafana, Prometheus, Loki, Tempo | evidence/attachments.txt, s3-checks.txt, idempotency.txt, session-revocation.txt, variants.txt, queue-recovery.txt, faults.txt, worker-load.json, upload-memory.json | +| Restart всех stores и телеметрии с volumes | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; времена запуска postgres, valkey, broker, s3, prometheus, loki, tempo, collector, grafana, api, lb, janitor, worker до и после | evidence/persistence.txt | +| Failover API-реплик, rolling update | `make test-failover`, `tests/scenario.py` | PASS для обеих реплик, 0 неуспешных попыток в публичном тесте, запись восстанавливается за 3.5 с | evidence/failover-1.txt, failover-2.txt, failover-each.json, rolling.json | +| Нагрузка | `tests/load.py --duration 30 --concurrency 4` | 11 666 успешных запросов, 387 RPS, 0 ошибок, p50/p95/p99 5.8 / 44.8 / 55.5 мс | evidence/load.json | +| Метрики, логи, трассы API и worker; низкая кардинальность; нет секретов | `tests/observability.py` | 13 сигналов есть; в labels нет user_id/chat_id/message_id/attachment_id/request_id; route — шаблон; тело сообщения и токен отсутствуют в Loki и Tempo | evidence/observability.txt | +| Запрос → trace → лог; job → worker trace → лог | `tests/observability.py` | по `request_id` лог API содержит `trace_id`/`span_id`; в Tempo trace из 30 spans; лог worker `job ready` с тем же `trace_id` | evidence/observability.txt | +| W3C trace context API → broker → worker | `tests/observability.py` | server → producer (родитель — запрос) → consumer (родитель — producer) → `s3 get`, `image render`, 4×`s3 put`, `db COMMIT`, `s3 remove` в одном trace | evidence/observability.txt | +| Цепочка по одному upload включая retry | `tests/observability.py` | S3 остановлен: 5 неуспешных попыток; после запуска S3 и `replay` попытка 6 → ready; 6 consumer и 6 producer spans в одном trace | evidence/observability.txt | +| Provisioning, импорт dashboard | `tests/dashboard.py` | 3 datasources и dashboard из Git, 28 панелей, 34 запроса без ошибок | evidence/dashboard.txt | +| Percentiles из histogram, доля ошибок, queue age, DB pool, saturation worker | dashboard (запросы выше) | панели показывают перечисленные величины | раздел «Dashboard», evidence/dashboard.txt | +| SLI/SLO и alerts, проверка искусственным отказом | инциденты 1 и 2 | firing, затем resolved | evidence/incident-worker.txt, incident-database.txt | +| Collector memory_limiter/batch, ограниченная буферизация, остановка хранилищ | `tests/incidents.py --only storage` | Prometheus, Loki, Tempo остановлены ≈ 80 с под нагрузкой: 942 сообщения 201 и 99 загрузок без 5xx; 3/3 вложения ready; RSS Collector 64 → 71 MiB; потери посчитаны | evidence/incident-storage.txt | +| История телеметрии на volumes | `tests/telemetry_persistence.py` | после перезапуска Prometheus, Loki, Tempo trace, его лог и метрики на месте | evidence/telemetry-persistence.txt | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Стенд: один хост, Docker 29, лимиты контейнеров из `compose.yaml`. Нагрузка — closed-loop клиент (3–4 потока, `Idempotency-Key`), в инцидентах плюс загрузка PNG раз в секунду. Числа ниже взяты из файлов evidence. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +### Инцидент 1. Остановлен worker + +Действие: `docker compose stop worker`, нагрузка загрузками. + +Диагностика только по сохранённым данным: +1. Alert `QueueLagGrowing` перешёл в firing через 60 с, возраст старейшей задачи 58 с, `queue_depth` 57. +2. Панели «Jobs per second by outcome» и «Worker saturation»: `jobs_processed_total` не растёт (0 задач/с), `worker_busy` равен 0. +3. Loki: за последнюю минуту нет ни одной записи `job ready` worker. +4. Tempo: trace свежей загрузки содержит server span API и producer span `attachments.jobs publish` и не содержит consumer span. Задача опубликована, её никто не забирает. + +Вывод: причина в остановленном worker, а не в API или брокере. Восстановление: `up -d worker`; alert разрешился через 20 с, очередь опустела за 21 с, потерь нет. + +### Инцидент 2. Заблокированная БД + +Действие: сессия psql держит `lock table messages in access exclusive mode` 45 с, нагрузка сообщениями. + +Диагностика: +1. `ApiHighErrorRate` в firing через 52 с, доля 5xx 0.47. +2. Метрики по маршрутам: 5xx только у `/api/v1/chats/{chat_id}/messages`. p99 ≈ 4.9 с. +3. Tempo, `{ status = error }`: server span 3006 мс, единственный долгий дочерний span `db INSERT` 2997 мс (≈ 100%) со статусом error. Время тратится в запросе к PostgreSQL (таймаут 3 с в `dbCtx`), а не в обработчике. +4. DB pool: ожидания соединения нет (0.01 с/с), пул не исчерпан. Значит причина не в нехватке соединений, а в одном долгом запросе. + +Вывод: причина в блокировке таблицы `messages` в PostgreSQL. Медленный обработчик исключён: у него нет собственного долгого этапа. Восстановление: блокировка снята, alert разрешился через 40 с; за время опыта 52 запроса получили 503, 164 — 201. + +### Инцидент 3. Недоступны хранилища телеметрии + +Prometheus, Loki, Tempo остановлены на ≈ 80 с под нагрузкой. API без ошибок (942 × 201, 99 загрузок), 3 новых вложения ready, сообщение создано. Collector не перезапускался и не убит по памяти, RSS 64 → 71 MiB при лимите контейнера 256 MiB. Потери посчитаны: `otelcol_exporter_send_failed_*` вырос на 2443 spans, 254 лога, 456 точек метрик. Данные за время недоступности потеряны (очередь 100 батчей и повтор до 20 с их не удерживают), данные до остановки сохранены, новые логи и метрики после запуска хранятся. + +Порог `memory_limiter` (160 MiB) в этом опыте не достигался: ограничение обеспечили ограниченные очереди и отбрасывание. + +### Прочее + +| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Отказ каждой API-реплики под нагрузкой | запись продолжается | 7 и 2 неуспешные попытки, 280 и 289 RPS против 448 до отказа | возврат реплики ≈ 6.8 с, потерь нет | +| Rolling update | без простоя | 2 неуспешные попытки, 462 RPS | старые реплики не отвечают | +| Restart всех stores и телеметрии | данные и сессия сохраняются | PASS | потерь нет | +| Нагрузка worker 6×40 Мп + 20 малых | не более 2 задач, память в лимите | все ready за 13.1 с, пик RSS 608 MiB / 768 | без OOM | + +Накладные расходы телеметрии не измерялись (трек 5 не заявлен). ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Collector, Prometheus, Loki, Tempo, Grafana, broker, PostgreSQL, S3 и HAProxy — единичные экземпляры. Остановка Collector приводит к потере телеметрии на время простоя (SDK держит не больше 2048 записей на сигнал и повторяет экспорт до 8 с). +- Collector не имеет дисковой очереди. Данные за время недоступности хранилища теряются, потери видны в `otelcol_exporter_send_failed_*`. +- Alerts проверены на коротких окнах (1 мин). Многооконный burn-rate (трек 5) не реализован, SLO за 30 суток на стенде не накапливаются. +- Метрики очереди берутся из PostgreSQL, а не из длины stream брокера: stream не обрезается (`XTRIM` не реализован), его длина не равна глубине очереди. +- `service.instance.id` — идентификатор контейнера: после пересоздания он меняется, старые ряды остаются в Prometheus до окончания retention. +- Трассы не сэмплируются (записывается всё). Цену наблюдаемости не измеряли. +- Порты Grafana, Prometheus, Loki, Tempo привязаны к 127.0.0.1; Prometheus, Loki и Tempo без аутентификации: допустимо только для локального стенда. +- Секреты лежат в `./secrets` вне Git. Права 644 нужны для чтения процессами контейнеров, это допустимо только для локального стенда. diff --git a/attachments.go b/attachments.go new file mode 100644 index 0000000..bd78cd0 --- /dev/null +++ b/attachments.go @@ -0,0 +1,534 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "hash" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "log/slog" + "net/http" + "os" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +const ( + objectPrefix = "attachments/" + stagingPrefix = "staging/" + probeHead = 256 << 10 + tailLen = 12 +) + +var ( + pngMagic = []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'} + pngTail = []byte{0, 0, 0, 0, 'I', 'E', 'N', 'D', 0xae, 0x42, 0x60, 0x82} + jpegMagic = []byte{0xff, 0xd8, 0xff} + jpegTail = []byte{0xff, 0xd9} +) + +type limits struct { + maxBytes int64 + maxPixels int64 + uploadSlots int + maxAttempts int +} + +type attachment struct { + ID, ChatID, OwnerID, Filename, ContentType, SHA256, Key, Status string + ErrorCode *string + Size int64 + Created, Updated time.Time + Variants []variant +} + +type variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + Size int64 `json:"size_bytes"` + Width int `json:"width"` + Height int `json:"height"` + SHA256 string `json:"sha256"` + key string +} + +func (x *attachment) json() map[string]any { + variants := x.Variants + if variants == nil { + variants = []variant{} + } + return map[string]any{ + "id": x.ID, + "chat_id": x.ChatID, + "owner_id": x.OwnerID, + "filename": x.Filename, + "content_type": x.ContentType, + "size_bytes": x.Size, + "sha256": x.SHA256, + "status": x.Status, + "variants": variants, + "error_code": x.ErrorCode, + "created_at": formatTime(x.Created), + "updated_at": formatTime(x.Updated), + } +} + +type probe struct { + sum hash.Hash + size int64 + head []byte + tail []byte +} + +func newProbe() *probe { + return &probe{sum: sha256.New()} +} + +func (p *probe) Write(b []byte) (int, error) { + p.sum.Write(b) + p.size += int64(len(b)) + if room := probeHead - len(p.head); room > 0 { + if room > len(b) { + room = len(b) + } + p.head = append(p.head, b[:room]...) + } + p.tail = append(p.tail, b...) + if len(p.tail) > tailLen { + p.tail = append(p.tail[:0], p.tail[len(p.tail)-tailLen:]...) + } + return len(b), nil +} + +func (p *probe) digest() string { + return hex.EncodeToString(p.sum.Sum(nil)) +} + +var ( + errNotImage = errors.New("not an image") + errTooBig = errors.New("too large") +) + +func (p *probe) verify(contentType string, maxPixels int64) error { + cfg, format, err := image.DecodeConfig(bytes.NewReader(p.head)) + if err != nil { + return errNotImage + } + var tail []byte + switch contentType { + case "image/png": + if format != "png" { + return errNotImage + } + tail = pngTail + case "image/jpeg": + if format != "jpeg" { + return errNotImage + } + tail = jpegTail + } + if !bytes.HasSuffix(p.tail, tail) { + return errNotImage + } + if int64(cfg.Width)*int64(cfg.Height) > maxPixels { + return errTooBig + } + return nil +} + +func magicMatches(contentType string, head []byte) bool { + switch contentType { + case "image/png": + return bytes.HasPrefix(head, pngMagic) + case "image/jpeg": + return bytes.HasPrefix(head, jpegMagic) + } + return false +} + +func validFilename(name string) bool { + n := utf8.RuneCountInString(name) + if n < 1 || n > 255 || !utf8.ValidString(name) { + return false + } + for _, r := range name { + if r == '/' || r == '\\' || r == 0 || unicode.IsControl(r) { + return false + } + } + return true +} + +type trackedReader struct { + r io.Reader + err error +} + +func (t *trackedReader) Read(b []byte) (int, error) { + n, err := t.r.Read(b) + if err != nil && err != io.EOF && t.err == nil { + t.err = err + } + return n, err +} + +func isTooLarge(err error) bool { + var mbe *http.MaxBytesError + return errors.As(err, &mbe) +} + +func (a *App) dropObject(key string) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := a.store.Remove(ctx, key); err != nil { + a.log.Warn("object cleanup failed", "key", key, "err", err.Error()) + } +} + +func (a *App) handleUploadAttachment(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + chatID := r.PathValue("chat_id") + if !isUUID(chatID) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id") + return + } + ok, err := a.isMember(r.Context(), chatID, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if !ok { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + select { + case a.uploads <- struct{}{}: + defer func() { <-a.uploads }() + default: + w.Header().Set("Retry-After", "1") + a.writeErr(w, reqID, 429, "rate_limited", "too many concurrent uploads") + return + } + filename := r.URL.Query().Get("filename") + if !validFilename(filename) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid filename") + return + } + contentType := strings.ToLower(strings.TrimSpace(strings.Split(r.Header.Get("Content-Type"), ";")[0])) + if contentType != "image/png" && contentType != "image/jpeg" { + a.writeErr(w, reqID, 415, "unsupported_media_type", "Content-Type must be image/png or image/jpeg") + return + } + if r.ContentLength > a.limits.maxBytes { + a.writeErr(w, reqID, 413, "payload_too_large", "file too large") + return + } + body := &trackedReader{r: http.MaxBytesReader(w, r.Body, a.limits.maxBytes)} + head := make([]byte, 16) + n, _ := io.ReadFull(body, head) + if body.err != nil { + if isTooLarge(body.err) { + a.writeErr(w, reqID, 413, "payload_too_large", "file too large") + } else { + a.writeErr(w, reqID, 400, "invalid_request", "upload interrupted") + } + return + } + head = head[:n] + if !magicMatches(contentType, head) { + a.writeErr(w, reqID, 415, "unsupported_media_type", "content does not match image type") + return + } + + id := newUUID() + trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("attachment.id", id)) + key := stagingPrefix + id + pr := newProbe() + size := r.ContentLength + if size < 0 { + size = -1 + } + src := io.TeeReader(io.MultiReader(bytes.NewReader(head), body), pr) + putCtx, cancel := context.WithTimeout(r.Context(), 60*time.Second) + defer cancel() + if err := a.store.Put(putCtx, key, src, size, contentType); err != nil { + a.dropObject(key) + switch { + case isTooLarge(body.err): + a.writeErr(w, reqID, 413, "payload_too_large", "file too large") + case body.err != nil: + a.writeErr(w, reqID, 400, "invalid_request", "upload interrupted") + default: + a.log.ErrorContext(r.Context(), "s3 put failed", "request_id", reqID, "err", err.Error()) + a.writeErr(w, reqID, 503, "unavailable", "storage unavailable") + } + return + } + if err := pr.verify(contentType, a.limits.maxPixels); err != nil { + a.dropObject(key) + if err == errTooBig { + a.writeErr(w, reqID, 413, "payload_too_large", "image too large") + } else { + a.writeErr(w, reqID, 415, "unsupported_media_type", "invalid image") + } + return + } + + now := nowUTC() + att := &attachment{ + ID: id, ChatID: chatID, OwnerID: u.ID, Filename: filename, ContentType: contentType, + SHA256: pr.digest(), Size: pr.size, Status: "queued", Created: now, Updated: now, + } + if err := a.enqueue(r.Context(), att, key); err != nil { + a.dropObject(key) + a.log.ErrorContext(r.Context(), "enqueue failed", "request_id", reqID, "err", err.Error()) + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if a.fault == "after_commit" { + a.log.Error("fault injected", "point", "after_commit", "attachment_id", att.ID) + os.Exit(3) + } + a.writeJSON(w, 202, att.json()) +} + +func (a *App) enqueue(ctx context.Context, att *attachment, key string) error { + ctx, cancel := dbCtx(ctx) + defer cancel() + tx, err := a.db.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + if _, err := tx.Exec(ctx, ` + insert into attachments(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key, status, created_at, updated_at) + values($1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, 'queued', $9, $9)`, + att.ID, att.ChatID, att.OwnerID, att.Filename, att.ContentType, att.Size, att.SHA256, key, att.Created); err != nil { + return err + } + jobID := newUUID() + tc := injectTrace(ctx) + if _, err := tx.Exec(ctx, ` + insert into jobs(id, attachment_id, pipeline_version, status, attempt, max_attempts, next_attempt_at, created_at, updated_at, traceparent, tracestate) + values($1::uuid, $2::uuid, $3, 'queued', 1, $4, $5, $5, $5, nullif($6, ''), nullif($7, ''))`, + jobID, att.ID, pipelineVersion, a.limits.maxAttempts, att.Created, tc.traceparent, tc.tracestate); err != nil { + return err + } + if _, err := tx.Exec(ctx, ` + insert into outbox(job_id, attempt, available_at, created_at) values($1::uuid, 1, $2, $2)`, jobID, att.Created); err != nil { + return err + } + return tx.Commit(ctx) +} + +func (a *App) loadAttachment(ctx context.Context, id, userID string) (*attachment, error) { + var x attachment + err := a.db.QueryRow(ctx, ` + select a.id::text, a.chat_id::text, a.owner_id::text, a.filename, a.content_type, + a.size_bytes, a.sha256, coalesce(a.object_key, ''), a.status, a.error_code, a.created_at, a.updated_at + from attachments a + join chat_members m on m.chat_id = a.chat_id and m.user_id = $2::uuid + where a.id = $1::uuid`, id, userID). + Scan(&x.ID, &x.ChatID, &x.OwnerID, &x.Filename, &x.ContentType, &x.Size, &x.SHA256, &x.Key, &x.Status, &x.ErrorCode, &x.Created, &x.Updated) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if x.Status != "ready" { + return &x, nil + } + rows, err := a.db.Query(ctx, ` + select name, content_type, size_bytes, width, height, sha256, object_key + from attachment_variants where attachment_id = $1::uuid and pipeline_version = $2 + order by array_position(array['thumbnail', 'crop', 'optimized'], name)`, id, pipelineVersion) + if err != nil { + return nil, err + } + x.Variants, err = pgx.CollectRows(rows, func(row pgx.CollectableRow) (variant, error) { + var v variant + return v, row.Scan(&v.Name, &v.ContentType, &v.Size, &v.Width, &v.Height, &v.SHA256, &v.key) + }) + if err != nil { + return nil, err + } + return &x, nil +} + +func (a *App) attachmentFor(w http.ResponseWriter, r *http.Request) *attachment { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return nil + } + id := r.PathValue("attachment_id") + if !isUUID(id) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid attachment_id") + return nil + } + ctx, cancel := dbCtx(r.Context()) + defer cancel() + att, err := a.loadAttachment(ctx, id, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return nil + } + if att == nil { + a.writeErr(w, reqID, 404, "not_found", "attachment not found") + return nil + } + return att +} + +func (a *App) handleGetAttachment(w http.ResponseWriter, r *http.Request) { + if att := a.attachmentFor(w, r); att != nil { + a.writeJSON(w, 200, att.json()) + } +} + +func (a *App) handleDownloadAttachment(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + name := r.URL.Query().Get("variant") + if name == "" { + name = "original" + } + if name != "original" && name != "thumbnail" && name != "crop" && name != "optimized" { + a.writeErr(w, reqID, 400, "invalid_request", "invalid variant") + return + } + att := a.attachmentFor(w, r) + if att == nil { + return + } + if att.Status != "ready" { + a.writeErr(w, reqID, 409, "conflict", "attachment is not ready") + return + } + key, contentType := att.Key, att.ContentType + if name != "original" { + key = "" + for _, v := range att.Variants { + if v.Name == name { + key, contentType = v.key, v.ContentType + } + } + if key == "" { + a.writeErr(w, reqID, 404, "not_found", "variant not found") + return + } + } + ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) + defer cancel() + obj, info, err := a.store.Get(ctx, key) + if err != nil { + if isNoSuchKey(err) { + a.writeErr(w, reqID, 404, "not_found", "attachment not found") + return + } + a.log.ErrorContext(r.Context(), "s3 get failed", "request_id", reqID, "err", err.Error()) + a.writeErr(w, reqID, 503, "unavailable", "storage unavailable") + return + } + defer obj.Close() + first := make([]byte, 32<<10) + n, err := io.ReadFull(obj, first) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + a.log.ErrorContext(r.Context(), "s3 read failed", "request_id", reqID, "err", err.Error()) + a.writeErr(w, reqID, 503, "unavailable", "storage unavailable") + return + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", strconv.FormatInt(info.Size, 10)) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(200) + _, _ = w.Write(first[:n]) + _, _ = io.Copy(w, obj) +} + +var keepQueries = map[string]string{ + objectPrefix: ` + select k from unnest($1::text[]) k + where exists(select 1 from attachments where object_key = k) + or exists(select 1 from attachment_variants where object_key = k)`, + stagingPrefix: ` + select k from unnest($1::text[]) k + where exists(select 1 from attachments where staging_key = k and (status in ('queued', 'processing') + or (status = 'failed' and updated_at > now() - make_interval(secs => $2))))`, +} + +func cleanupOrphans(ctx context.Context, db *pgxpool.Pool, store *Store, grace, retention time.Duration, log *slog.Logger) (int, error) { + cutoff := time.Now().Add(-grace) + removed := 0 + flush := func(prefix string, keys []string) error { + if len(keys) == 0 { + return nil + } + args := []any{keys} + if prefix == stagingPrefix { + args = append(args, retention.Seconds()) + } + rows, err := db.Query(ctx, keepQueries[prefix], args...) + if err != nil { + return err + } + kept, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return err + } + known := map[string]bool{} + for _, k := range kept { + known[k] = true + } + for _, k := range keys { + if known[k] { + continue + } + if err := store.Remove(ctx, k); err != nil { + return err + } + log.Info("orphan object removed", "key", k) + removed++ + } + return nil + } + for _, prefix := range []string{objectPrefix, stagingPrefix} { + var batch []string + for obj := range store.List(ctx, prefix) { + if obj.Err != nil { + return removed, obj.Err + } + if obj.LastModified.After(cutoff) { + continue + } + batch = append(batch, obj.Key) + if len(batch) == 500 { + if err := flush(prefix, batch); err != nil { + return removed, err + } + batch = batch[:0] + } + } + if err := flush(prefix, batch); err != nil { + return removed, err + } + } + return removed, nil +} diff --git a/auth.go b/auth.go new file mode 100644 index 0000000..cacc42d --- /dev/null +++ b/auth.go @@ -0,0 +1,131 @@ +package main + +import ( + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" + "unicode/utf8" + + "golang.org/x/crypto/argon2" +) + +const ( + argonTime = 2 + argonMemory = 19456 + argonThreads = 1 + argonKeyLen = 32 + argonSaltLen = 16 +) + +func hashPassword(password string) (string, error) { + salt := make([]byte, argonSaltLen) + if _, err := rand.Read(salt); err != nil { + return "", err + } + hash := argon2.IDKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen) + b64Salt := base64.RawStdEncoding.EncodeToString(salt) + b64Hash := base64.RawStdEncoding.EncodeToString(hash) + return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", + argon2.Version, argonMemory, argonTime, argonThreads, b64Salt, b64Hash), nil +} + +func checkPassword(encoded, password string) (bool, error) { + parts := strings.Split(encoded, "$") + if len(parts) != 6 || parts[1] != "argon2id" { + return false, errors.New("bad hash") + } + var version int + if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil { + return false, err + } + var memory, timeCost, threads int + for _, p := range strings.Split(parts[3], ",") { + kv := strings.SplitN(p, "=", 2) + if len(kv) != 2 { + return false, errors.New("bad params") + } + n, err := strconv.Atoi(kv[1]) + if err != nil { + return false, err + } + switch kv[0] { + case "m": + memory = n + case "t": + timeCost = n + case "p": + threads = n + } + } + salt, err := base64.RawStdEncoding.DecodeString(parts[4]) + if err != nil { + return false, err + } + want, err := base64.RawStdEncoding.DecodeString(parts[5]) + if err != nil { + return false, err + } + got := argon2.IDKey([]byte(password), salt, uint32(timeCost), uint32(memory), uint8(threads), uint32(len(want))) + return subtle.ConstantTimeCompare(got, want) == 1, nil +} + +func newToken() (string, []byte, string, error) { + raw := make([]byte, 32) + if _, err := rand.Read(raw); err != nil { + return "", nil, "", err + } + token := base64.RawURLEncoding.EncodeToString(raw) + sum := sha256.Sum256([]byte(token)) + hash := make([]byte, len(sum)) + copy(hash, sum[:]) + return token, hash, hex.EncodeToString(hash), nil +} + +func tokenHash(token string) ([]byte, string) { + sum := sha256.Sum256([]byte(token)) + hash := make([]byte, len(sum)) + copy(hash, sum[:]) + return hash, hex.EncodeToString(hash) +} + +func parseBasic(header string) (username, password string, ok bool) { + const prefix = "Basic " + if !strings.HasPrefix(header, prefix) { + return "", "", false + } + decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(header[len(prefix):])) + if err != nil { + return "", "", false + } + s := string(decoded) + i := strings.IndexByte(s, ':') + if i < 0 { + return "", "", false + } + username = s[:i] + password = s[i+1:] + return username, password, true +} + +func parseBearer(header string) (string, bool) { + const prefix = "Bearer " + if !strings.HasPrefix(header, prefix) { + return "", false + } + tok := strings.TrimSpace(header[len(prefix):]) + if tok == "" { + return "", false + } + return tok, true +} + +func validPassword(p string) bool { + n := utf8.RuneCountInString(p) + return n >= 12 && n <= 128 +} diff --git a/broker.go b/broker.go new file mode 100644 index 0000000..fea8e4b --- /dev/null +++ b/broker.go @@ -0,0 +1,162 @@ +package main + +import ( + "context" + "encoding/json" + "log/slog" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +const ( + jobStream = "attachments.jobs" + jobGroup = "workers" + jobType = "attachment.process.v1" + pipelineVersion = 1 +) + +type jobMessage struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt string `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +type Broker struct { + rdb *redis.Client +} + +func openBroker() (*Broker, error) { + raw, err := readSecret("BROKER_URL", "BROKER_URL_FILE") + if err != nil { + return nil, err + } + opt, err := redis.ParseURL(raw) + if err != nil { + return nil, err + } + opt.DialTimeout = 2 * time.Second + opt.ReadTimeout = 3 * time.Second + opt.WriteTimeout = 3 * time.Second + opt.MaxRetries = 0 + return &Broker{rdb: redis.NewClient(opt)}, nil +} + +func (b *Broker) ensureGroup(ctx context.Context) error { + err := b.rdb.XGroupCreateMkStream(ctx, jobStream, jobGroup, "0").Err() + if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") { + return err + } + return nil +} + +func (b *Broker) publish(ctx context.Context, m jobMessage) error { + body, err := json.Marshal(m) + if err != nil { + return err + } + return b.rdb.XAdd(ctx, &redis.XAddArgs{Stream: jobStream, Values: map[string]any{"job": string(body)}}).Err() +} + +func (b *Broker) publishTraced(ctx context.Context, m jobMessage) (err error) { + if m.Traceparent == "" { + return b.publish(ctx, m) + } + ctx = extractTrace(ctx, traceCarrier{m.Traceparent, m.Tracestate}) + ctx, sp := tracer.Start(ctx, jobStream+" publish", trace.WithSpanKind(trace.SpanKindProducer), trace.WithAttributes( + attribute.String("messaging.system", "redis"), + attribute.String("messaging.destination.name", jobStream), + attribute.String("messaging.operation.type", "publish"), + attribute.String("job.id", m.JobID), + attribute.String("attachment.id", m.AttachmentID), + attribute.Int("job.attempt", m.Attempt), + )) + defer func() { endSpan(sp, err) }() + c := injectTrace(ctx) + m.Traceparent, m.Tracestate = c.traceparent, "" + if len(c.tracestate) <= 512 { + m.Tracestate = c.tracestate + } + return b.publish(ctx, m) +} + +func relayOnce(ctx context.Context, db *pgxpool.Pool, b *Broker) (int, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + tx, err := db.Begin(ctx) + if err != nil { + return 0, err + } + defer tx.Rollback(ctx) + rows, err := tx.Query(ctx, ` + select o.id, o.job_id::text, o.attempt, j.attachment_id::text, j.pipeline_version, j.created_at, + coalesce(j.traceparent, ''), coalesce(j.tracestate, '') + from outbox o + join jobs j on j.id = o.job_id + where o.published_at is null and o.available_at <= now() + order by o.id + limit 50 + for update of o skip locked`) + if err != nil { + return 0, err + } + var ids []int64 + var msgs []jobMessage + for rows.Next() { + var id int64 + var created time.Time + m := jobMessage{SchemaVersion: 1, Type: jobType} + if err := rows.Scan(&id, &m.JobID, &m.Attempt, &m.AttachmentID, &m.PipelineVersion, &created, &m.Traceparent, &m.Tracestate); err != nil { + rows.Close() + return 0, err + } + m.CreatedAt = formatTime(created) + ids = append(ids, id) + msgs = append(msgs, m) + } + rows.Close() + if err := rows.Err(); err != nil { + return 0, err + } + if len(ids) == 0 { + return 0, nil + } + if err := b.ensureGroup(ctx); err != nil { + return 0, err + } + for _, m := range msgs { + if err := b.publishTraced(ctx, m); err != nil { + return 0, err + } + } + if _, err := tx.Exec(ctx, `update outbox set published_at = now() where id = any($1)`, ids); err != nil { + return 0, err + } + return len(ids), tx.Commit(ctx) +} + +func runRelay(ctx context.Context, db *pgxpool.Pool, b *Broker, log *slog.Logger) { + for ctx.Err() == nil { + n, err := relayOnce(ctx, db, b) + if err != nil { + log.Warn("outbox relay failed", "err", err.Error()) + time.Sleep(2 * time.Second) + continue + } + if n > 0 { + log.Info("outbox published", "count", n) + continue + } + time.Sleep(300 * time.Millisecond) + } +} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..2f4a5d3 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,475 @@ +x-api-env: &api-env + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + SESSION_STORE: postgres + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-10} + GOMEMLIMIT: 150MiB + SHUTDOWN_TIMEOUT_SECONDS: 10 + DRAIN_SECONDS: 3 + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + BROKER_URL_FILE: /run/secrets/broker_url + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + FAULT_POINT: ${API_FAULT_POINT:-} + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${APP_ENV:-lab} + +services: + postgres: + image: postgres:17-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24 + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_DB: messenger + secrets: + - postgres_password + volumes: + - pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + networks: + - backend + restart: unless-stopped + + valkey: + image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: ["valkey-server", "--appendonly", "yes", "--save", "60", "1"] + volumes: + - valkey_data:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: unless-stopped + + broker: + image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: ["valkey-server", "/run/secrets/broker_conf", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory-policy", "noeviction"] + secrets: + - broker_conf + volumes: + - broker_data:/data + healthcheck: + test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$(sed -n 's/^requirepass //p' /run/secrets/broker_conf)\" ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: unless-stopped + + s3: + image: chrislusf/seaweedfs@sha256:c2658820e763294a9e9de7b36c81e59b879e3459148351aee4e31a126ca64860 + command: + - server + - -dir=/data + - -ip=s3 + - -master.volumeSizeLimitMB=128 + - -volume.max=30 + - -s3 + - -s3.port=9000 + - -s3.config=/run/secrets/s3_config + secrets: + - s3_config + volumes: + - s3_data:/data + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9333/dir/assign | grep -q fid"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + networks: + - storage + restart: unless-stopped + + s3-init: + image: mtusi-messenger:lab6 + command: ["s3-init"] + environment: + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + depends_on: + s3: + condition: service_healthy + migrate: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - storage + restart: "no" + + janitor: + image: mtusi-messenger:lab6 + command: ["janitor"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + CLEANUP_GRACE_SECONDS: ${CLEANUP_GRACE_SECONDS:-600} + CLEANUP_INTERVAL_SECONDS: ${CLEANUP_INTERVAL_SECONDS:-300} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3-init: + condition: service_completed_successfully + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.25" + memory: 64M + networks: + - backend + - storage + restart: unless-stopped + + worker: + image: mtusi-messenger:lab6 + command: ["worker"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_BACKOFF_BASE_MS: ${JOB_BACKOFF_BASE_MS:-1000} + JOB_BACKOFF_MAX_MS: ${JOB_BACKOFF_MAX_MS:-30000} + GOMEMLIMIT: 600MiB + FAULT_POINT: ${WORKER_FAULT_POINT:-} + OTEL_SERVICE_NAME: messenger-worker + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${APP_ENV:-lab} + secrets: + - database_url + - broker_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3-init: + condition: service_completed_successfully + broker: + condition: service_healthy + otel-collector: + condition: service_started + healthcheck: + disable: true + stop_grace_period: 30s + deploy: + resources: + limits: + cpus: "1.0" + memory: 768M + networks: + - backend + - storage + restart: unless-stopped + + migrate: + build: . + image: mtusi-messenger:lab6 + command: ["migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: "no" + + api: + image: mtusi-messenger:lab6 + environment: *api-env + secrets: + - database_url + - cache_url + - broker_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3-init: + condition: service_completed_successfully + valkey: + condition: service_healthy + broker: + condition: service_healthy + otel-collector: + condition: service_started + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "0.5" + memory: 192M + networks: + - backend + - storage + restart: unless-stopped + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.116.1@sha256:d0ebf65280da2e1b1491d1b93648281afd353d4b9ea19160090303cec9a233bd + command: ["--config=/etc/otelcol/config.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro + depends_on: + prometheus: + condition: service_healthy + loki: + condition: service_healthy + tempo: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + networks: + - backend + - telemetry + restart: unless-stopped + + prometheus: + image: prom/prometheus:v3.1.0@sha256:6559acbd5d770b15bb3c954629ce190ac3cbbdb2b7f1c30f0385c4e05104e218 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + - --web.enable-otlp-receiver + ports: + - "127.0.0.1:9090:9090" + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/rules:/etc/prometheus/rules:ro + - prometheus_data:/prometheus + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/ready"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.5" + memory: 384M + networks: + - telemetry + restart: unless-stopped + + loki: + image: grafana/loki:3.3.2@sha256:8af2de1abbdd7aa92b27c9bcc96f0f4140c9096b507c77921ffddf1c6ad6c48f + command: ["-config.file=/etc/loki/loki.yaml"] + ports: + - "127.0.0.1:3100:3100" + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - loki_data:/loki + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3100/ready"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 15s + deploy: + resources: + limits: + cpus: "0.5" + memory: 384M + networks: + - telemetry + restart: unless-stopped + + tempo: + image: grafana/tempo:2.7.0@sha256:12e904cc509a8e0bd87815e0bc719b6167933bc3f2532716a85811af967780f6 + command: ["-config.file=/etc/tempo/tempo.yaml"] + ports: + - "127.0.0.1:3200:3200" + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo_data:/var/tempo + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3200/ready"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 15s + deploy: + resources: + limits: + cpus: "0.5" + memory: 384M + networks: + - telemetry + restart: unless-stopped + + grafana: + image: grafana/grafana:11.3.0@sha256:a0f881232a6fb71a0554a47d0fe2203b6888fe77f4cefb7ea62bed7eb54e13c3 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password + GF_USERS_ALLOW_SIGN_UP: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + secrets: + - grafana_admin_password + ports: + - "127.0.0.1:3000:3000" + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + depends_on: + prometheus: + condition: service_healthy + loki: + condition: service_healthy + tempo: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 15s + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + networks: + - telemetry + restart: unless-stopped + + lb: + image: haproxy:3.0-alpine@sha256:cf3af0facf7eb22b9fee463daf0fbc07ead740b8150696dce724264faf7b4318 + ports: + - "127.0.0.1:8080:8080" + volumes: + - ./lb/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + - api + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: unless-stopped + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + broker_conf: + file: ./secrets/broker.conf + broker_url: + file: ./secrets/broker_url + s3_config: + file: ./secrets/s3_config.json + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + grafana_admin_password: + file: ./secrets/grafana_admin_password + +volumes: + pg_data: + valkey_data: + broker_data: + s3_data: + prometheus_data: + loki_data: + tempo_data: + grafana_data: + +networks: + backend: + driver: bridge + storage: + driver: bridge + telemetry: + driver: bridge diff --git a/db.go b/db.go new file mode 100644 index 0000000..371c9bf --- /dev/null +++ b/db.go @@ -0,0 +1,59 @@ +package main + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func readSecret(envKey, fileKey string) (string, error) { + if v := os.Getenv(envKey); v != "" { + return strings.TrimSpace(v), nil + } + path := os.Getenv(fileKey) + if path == "" { + return "", fmt.Errorf("%s or %s required", envKey, fileKey) + } + b, err := os.ReadFile(path) + if err != nil { + return "", err + } + return strings.TrimSpace(string(b)), nil +} + +func openDB(ctx context.Context) (*pgxpool.Pool, error) { + dsn, err := readSecret("DATABASE_URL", "DATABASE_URL_FILE") + if err != nil { + return nil, err + } + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + return nil, err + } + cfg.MaxConns = 10 + cfg.MinConns = 2 + cfg.MaxConnLifetime = 5 * time.Minute + cfg.ConnConfig.ConnectTimeout = 3 * time.Second + cfg.ConnConfig.Tracer = dbTracer{} + + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + return nil, err + } + pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := pool.Ping(pingCtx); err != nil { + pool.Close() + return nil, err + } + registerPoolMetrics(pool) + return pool, nil +} + +func dbCtx(parent context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(parent, 3*time.Second) +} diff --git a/evidence/attachments.txt b/evidence/attachments.txt new file mode 100644 index 0000000..6e382b8 --- /dev/null +++ b/evidence/attachments.txt @@ -0,0 +1 @@ +PASS: attachment validation, limits and access control diff --git a/evidence/dashboard.txt b/evidence/dashboard.txt new file mode 100644 index 0000000..b037e6b --- /dev/null +++ b/evidence/dashboard.txt @@ -0,0 +1,2 @@ +grafana: 3 datasources and dashboard "Messenger observability" provisioned from Git, 28 panels, 34 queries executed without errors +PASS: provisioning and dashboard queries diff --git a/evidence/failover-1.txt b/evidence/failover-1.txt new file mode 100644 index 0000000..96cc3b2 --- /dev/null +++ b/evidence/failover-1.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 138, "failed_attempts": 0, "first_recovery_seconds": 3.483} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-2.txt b/evidence/failover-2.txt new file mode 100644 index 0000000..1967b2c --- /dev/null +++ b/evidence/failover-2.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 138, "failed_attempts": 0, "first_recovery_seconds": 3.506} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-each.json b/evidence/failover-each.json new file mode 100644 index 0000000..ab270eb --- /dev/null +++ b/evidence/failover-each.json @@ -0,0 +1,71 @@ +{ + "baseline": { + "window_seconds": 6, + "successful_requests": 2689, + "failed_attempts": 0, + "successful_rps": 448.2, + "last_error_offset_seconds": null, + "instances": [ + "1dbf9e0d259e", + "638505d2d88b" + ] + }, + "replicas": [ + { + "stopped_replica": "638505d2d88b", + "during_failure": { + "window_seconds": 15.37, + "successful_requests": 4298, + "failed_attempts": 7, + "successful_rps": 279.7, + "last_error_offset_seconds": 1.06, + "instances": [ + "1dbf9e0d259e", + "638505d2d88b" + ] + }, + "seen_stopped_replica_after_stop": false, + "stop_command_seconds": 3.37, + "restore_seconds": 6.79, + "after_restore": { + "window_seconds": 7.0, + "successful_requests": 3123, + "failed_attempts": 0, + "successful_rps": 446.1, + "last_error_offset_seconds": null, + "instances": [ + "1dbf9e0d259e", + "638505d2d88b" + ] + } + }, + { + "stopped_replica": "1dbf9e0d259e", + "during_failure": { + "window_seconds": 15.49, + "successful_requests": 4470, + "failed_attempts": 2, + "successful_rps": 288.5, + "last_error_offset_seconds": 1.04, + "instances": [ + "1dbf9e0d259e", + "638505d2d88b" + ] + }, + "seen_stopped_replica_after_stop": false, + "stop_command_seconds": 3.49, + "restore_seconds": 6.8, + "after_restore": { + "window_seconds": 7.0, + "successful_requests": 3129, + "failed_attempts": 0, + "successful_rps": 447.0, + "last_error_offset_seconds": null, + "instances": [ + "1dbf9e0d259e", + "638505d2d88b" + ] + } + } + ] +} diff --git a/evidence/faults.txt b/evidence/faults.txt new file mode 100644 index 0000000..e65dbdb --- /dev/null +++ b/evidence/faults.txt @@ -0,0 +1,7 @@ +1) api crashed after COMMIT before 202/publish: attachment=8bcd0ac3-c34d-45d0-8f48-55a0283f7832 recovered by outbox relay (published rows=1) -> ready attempt=1 +2) worker crashed after S3 write before COMMIT: lease expired, reaper requeued, attempt=2 ready, variants=3 +3) worker crashed after COMMIT before ack: message redelivered after restart, skipped as no-op, attempt=1, variants=3 +4) s3 down: transient errors retried with backoff+jitter delays_ms=[895, 1219, 3028, 5245], then failed retry_exhausted, listed in registry + replay: failed -> queued -> ready (audit rows=1); replay of ready job skipped, variants=3 +5) corrupted IDAT (passes header/tail check in API): failed invalid_image on attempt 1, no retries +PASS: outbox recovery, lease recovery, redelivery after commit, backoff with jitter, retry_exhausted, replay, permanent error diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt new file mode 100644 index 0000000..50987a9 --- /dev/null +++ b/evidence/idempotency.txt @@ -0,0 +1 @@ +PASS: 32 parallel requests, 1 message, 2 instances diff --git a/evidence/incident-database.txt b/evidence/incident-database.txt new file mode 100644 index 0000000..5a7943b --- /dev/null +++ b/evidence/incident-database.txt @@ -0,0 +1,4 @@ +2) database lock on messages: ApiHighErrorRate firing after 52s; 5xx ratio=0.47 only on ['/api/v1/chats/{chat_id}/messages']; p99=4.9s; pool used=0.0 acquire wait=0.01 s/s + diagnosis from stored data: error trace 17a3258b1897cd36f88418fb1adc23c5, server span 3006 ms, child "db INSERT" 2997 ms (100%) with status error, handler has no other slow child; requests: {503: 52, 201: 164} + restored: lock released, alert resolved after 40s +PASS: slow database detected by alert and diagnosed from telemetry diff --git a/evidence/incident-storage.txt b/evidence/incident-storage.txt new file mode 100644 index 0000000..406fb1d --- /dev/null +++ b/evidence/incident-storage.txt @@ -0,0 +1,5 @@ +3) prometheus, loki and tempo stopped for ~80s under load: requests={201: 942, 'upload': 99}, 3/3 uploads ready, message 201, no 5xx + collector rss MiB: start=64 peak=71 during outage (limit 256, memory_limiter 160), restarts/oom=0 false + telemetry lost and counted: collector send_failed spans +2443, log records +254, metric points +456; sdk export errors=0 + after restore new logs and metrics are stored again; data from the outage window is lost, data before it remains +PASS: telemetry storage outage does not affect the API and loss is bounded and counted diff --git a/evidence/incident-worker.txt b/evidence/incident-worker.txt new file mode 100644 index 0000000..55bc029 --- /dev/null +++ b/evidence/incident-worker.txt @@ -0,0 +1,4 @@ +1) worker stopped: QueueLagGrowing firing after 60s; oldest job age=58s depth=57 jobs/s=0.00 worker_busy=0.0 + diagnosis from stored data: trace 4710543929ec0f33fc13ce8565b5bc6a has the api server and "attachments.jobs publish" spans and no consumer span; no "job ready" worker log in Loki during the last minute (0 lines) + restored: worker started, alert resolved after 20s, backlog drained in 21s +PASS: stopped worker detected by alert and diagnosed from telemetry diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 0000000..97ecea3 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.116, + "concurrency": 4, + "successful_requests": 11666, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 387.368, + "successful_latency_ms": { + "p50": 5.802, + "p95": 44.802, + "p99": 55.465 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/observability.txt b/evidence/observability.txt new file mode 100644 index 0000000..15947bd --- /dev/null +++ b/evidence/observability.txt @@ -0,0 +1,5 @@ +chain: request_id=77a5a56d-55b7-4fb1-9849-a4c39d03d114 trace_id=391ac5aeb8757b980131897c9b96322b spans=30 api->producer->consumer->s3/db linked, api and worker logs carry trace_id +metrics: 13 signals present, labels=['deployment_environment_name', 'http_request_method', 'http_response_status_class', 'http_route', 'instance', 'job', 'job_type', 'le', 'outcome', 'queue', 'service_version', 'state'], routes are templates (10) +secrets: message body and session token absent from Loki and Tempo +retry: one trace 9bb4fe6dc99b3dbde778e1758a85acd9 holds 6 attempts (5 failed with S3 down, attempt 6 ready after replay) +PASS: signals, producer/consumer trace chain, log correlation, retry in trace, no secrets in telemetry diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..74e1ab5 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,32 @@ +== before +postgres 2026-11-19T19:48:24.393646212Z +valkey 2026-11-19T19:48:24.279274086Z +broker 2026-11-19T19:48:24.52160412Z +s3 2026-11-19T19:48:34.104469144Z +prometheus 2026-11-19T19:48:24.131549676Z +loki 2026-11-19T19:48:25.768218547Z +tempo 2026-11-19T19:48:25.585571452Z +otel-collector 2026-11-19T19:49:10.223513181Z +grafana 2026-11-19T19:49:10.079005478Z +api 2026-11-19T19:49:13.028782328Z +api 2026-11-19T19:49:13.082849306Z +lb 2026-11-19T19:49:10.26422772Z +janitor 2026-11-19T19:49:10.239020362Z +worker 2026-11-19T19:49:11.105495834Z +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +== after +postgres 2026-11-19T19:49:23.902795389Z +valkey 2026-11-19T19:49:23.736355641Z +broker 2026-11-19T19:49:23.770551967Z +s3 2026-11-19T19:49:33.778194628Z +prometheus 2026-11-19T19:49:23.880621282Z +loki 2026-11-19T19:49:25.646714025Z +tempo 2026-11-19T19:49:24.313008245Z +otel-collector 2026-11-19T19:50:10.305030304Z +grafana 2026-11-19T19:50:10.322614526Z +api 2026-11-19T19:50:13.127529668Z +api 2026-11-19T19:50:13.147392359Z +lb 2026-11-19T19:50:10.154157227Z +janitor 2026-11-19T19:50:10.272224264Z +worker 2026-11-19T19:50:14.68861531Z diff --git a/evidence/queue-recovery.txt b/evidence/queue-recovery.txt new file mode 100644 index 0000000..5cf545f --- /dev/null +++ b/evidence/queue-recovery.txt @@ -0,0 +1,5 @@ +accepted: attachment=e91cf4d7-c48d-43cd-9a1e-633ca1dabb90 job=ca2f9db6-c303-4663-b204-eb589ee11616 status=queued, job in stream, worker stopped +restarted broker and api: job still in stream, attachment still queued +worker started: attachment ready, attempt=1, variants=3 +duplicate delivery: skipped as no-op, acked, variants=3, updated_at unchanged +PASS: worker stop, broker/api restart, redelivery idempotency diff --git a/evidence/rolling.json b/evidence/rolling.json new file mode 100644 index 0000000..3137293 --- /dev/null +++ b/evidence/rolling.json @@ -0,0 +1,24 @@ +{ + "old_replicas": [ + "638505d2d88b", + "1dbf9e0d259e" + ], + "new_replicas": [ + "02a35d2fcbe0", + "20278f44f3e4" + ], + "update_window": { + "window_seconds": 16.31, + "successful_requests": 7541, + "failed_attempts": 2, + "successful_rps": 462.3, + "last_error_offset_seconds": 12.63, + "instances": [ + "02a35d2fcbe0", + "1dbf9e0d259e", + "20278f44f3e4", + "638505d2d88b" + ] + }, + "old_replicas_served_after_update": [] +} diff --git a/evidence/s3-checks.txt b/evidence/s3-checks.txt new file mode 100644 index 0000000..6964887 --- /dev/null +++ b/evidence/s3-checks.txt @@ -0,0 +1 @@ +PASS: private bucket, limited credentials, s3 outage, orphan and staging cleanup diff --git a/evidence/session-revocation.txt b/evidence/session-revocation.txt new file mode 100644 index 0000000..38dab27 --- /dev/null +++ b/evidence/session-revocation.txt @@ -0,0 +1 @@ +PASS: token revoked while valkey was stopped stays revoked after valkey restart (401) diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..566c729 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,4 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/telemetry-persistence.txt b/evidence/telemetry-persistence.txt new file mode 100644 index 0000000..4b8ef5a --- /dev/null +++ b/evidence/telemetry-persistence.txt @@ -0,0 +1,2 @@ +persistence: trace c23eec016505cb2f6cd5dd7d6b400c15 (30 spans), its request log and 18 metric series survived a restart of prometheus, loki and tempo +PASS: telemetry history is kept on volumes diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 0000000..e9ff765 --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,18 @@ +{ + "file_bytes": 9722608, + "concurrent_uploads": 12, + "rounds": 3, + "uploads_ok": 36, + "uploads_failed": 0, + "elapsed_seconds": 18.72, + "api_memory_limit_mib": 192, + "idle_memory_mib": { + "mtusi2026lab6-fork-api-1": 67.2, + "mtusi2026lab6-fork-api-2": 70.4 + }, + "peak_memory_mib": { + "mtusi2026lab6-fork-api-1": 115.8, + "mtusi2026lab6-fork-api-2": 135.4 + }, + "buffering_all_files_would_need_mib": 111.3 +} diff --git a/evidence/variants.txt b/evidence/variants.txt new file mode 100644 index 0000000..9a5e8bf --- /dev/null +++ b/evidence/variants.txt @@ -0,0 +1 @@ +PASS: thumbnail/crop/optimized sizes, formats, checksums, no upscale, alpha on white, EXIF orientation, no EXIF in variants diff --git a/evidence/worker-load.json b/evidence/worker-load.json new file mode 100644 index 0000000..0415a8b --- /dev/null +++ b/evidence/worker-load.json @@ -0,0 +1,26 @@ +{ + "huge_images": { + "count": 6, + "pixels": 39942400, + "bytes": 116545 + }, + "small_images": 20, + "worker_concurrency": 2, + "max_processing_jobs_observed": 2, + "all_ready": true, + "accept_seconds": 0.35, + "all_ready_seconds": 13.1, + "worker_peak_rss_mib": 607.7, + "worker_memory_limit_mib": 768, + "worker_restarts_oom": [ + "0", + "false" + ], + "timeout_case": { + "job_timeout_seconds": 1, + "status": "failed", + "error_code": "retry_exhausted", + "attempts": 5, + "last_error": "context deadline exceeded" + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d4b4159 --- /dev/null +++ b/go.mod @@ -0,0 +1,61 @@ +module mtusi-messenger + +go 1.26.0 + +require ( + github.com/gen2brain/webp v0.6.4 + github.com/jackc/pgx/v5 v5.11.0 + github.com/minio/minio-go/v7 v7.4.1 + github.com/redis/go-redis/v9 v9.23.0 + github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd + go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 + go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 + go.opentelemetry.io/otel/metric v1.47.0 + go.opentelemetry.io/otel/sdk v1.47.0 + go.opentelemetry.io/otel/sdk/log v1.47.0 + go.opentelemetry.io/otel/sdk/metric v1.47.0 + go.opentelemetry.io/otel/trace v1.47.0 + golang.org/x/crypto v0.58.0 + golang.org/x/image v0.47.0 +) + +require ( + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.10.1 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect + go.uber.org/atomic v1.12.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/net v0.60.0 // indirect + golang.org/x/sync v0.24.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/text v0.43.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/grpc v1.83.2 // indirect + google.golang.org/protobuf v1.36.12 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..fa49710 --- /dev/null +++ b/go.sum @@ -0,0 +1,142 @@ +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/gen2brain/webp v0.6.4 h1:SUDdmxADOAiPQ+5ylNmuHhuYf2dOi0KgKZHL5vpVCNU= +github.com/gen2brain/webp v0.6.4/go.mod h1:iGWMaCSw7t3I/Cv9llzEKmpnR36S8lS8VL/ZVjxU0JE= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w= +github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= +github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 h1:/CdgE52i1OEPHhsGUqrggYNRaZy4v4YgVenGd52dRBI= +go.opentelemetry.io/contrib/bridges/otelslog v0.21.0/go.mod h1:9pBdbVbuehIDzxN4accnzzxEUPT8Nq7grbZMo+J1VN4= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0 h1:5WVp40k2fJHuc5BtNTgKk+EugKFT8Eg3lywFfCPI/nE= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0/go.mod h1:59nH0RYCGHRlwt15WyioeSYINBbuObeBpTowPhEKWHY= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 h1:7kszRN4Ze5GeAkUnBjBSUIunVBMA+mZIG04auueqguM= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0/go.mod h1:CLzakYy8jOdkBfcveCWYkeKJxujcF4xPhh2Z7IfE98U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM= +go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/log v1.47.0 h1:W4N/ZfgpkRRnmARnqYoQ6BrWA0mrj8IzVDNLlpmUJZY= +go.opentelemetry.io/otel/sdk/log v1.47.0/go.mod h1:Wu74Z2EmOKXi2FleSNAb0Hyau2G+r5kubSEMJw5aAMs= +go.opentelemetry.io/otel/sdk/log/logtest v0.23.0 h1:j3pm42QCrKYYLLkI6OrV1QHFkUfDA6/4EP+Y1PzggLg= +go.opentelemetry.io/otel/sdk/log/logtest v0.23.0/go.mod h1:zt0gEnuTZgDvqvczHwYswz93eqMaIr6upTTsdV/HUlU= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= +go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo= +golang.org/x/image v0.47.0/go.mod h1:IkN5o4A7vq55vXN8FWjjnhg9Rs2GOtffqsSi9mWAKYE= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/handlers.go b/handlers.go new file mode 100644 index 0000000..2af3b48 --- /dev/null +++ b/handlers.go @@ -0,0 +1,1095 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "regexp" + "strconv" + "strings" + "sync/atomic" + "time" + "unicode/utf8" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" +) + +var idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +var instanceID = func() string { + h, err := os.Hostname() + if err != nil || h == "" { + return newUUID() + } + return h +}() + +var usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +type apiError struct { + Code string `json:"code"` + Message string `json:"message"` + ReqID string `json:"request_id"` +} + +type errorBody struct { + Error apiError `json:"error"` +} + +type App struct { + db *pgxpool.Pool + log *slog.Logger + ready atomic.Bool + sessionTTL time.Duration + idempotencyTTL time.Duration + store *Store + limits limits + uploads chan struct{} + fault string +} + +func NewApp(db *pgxpool.Pool, log *slog.Logger, sessionTTL, idempotencyTTL time.Duration, store *Store, lim limits) *App { + a := &App{db: db, log: log, sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, store: store, limits: lim, uploads: make(chan struct{}, lim.uploadSlots), fault: os.Getenv("FAULT_POINT")} + a.ready.Store(true) + return a +} + +func (a *App) SetReady(v bool) { + a.ready.Store(v) +} + +func newUUID() string { + var b [16]byte + _, _ = rand.Read(b[:]) + b[6] = (b[6] & 0x0f) | 0x40 + b[8] = (b[8] & 0x3f) | 0x80 + return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]) +} + +func nowUTC() time.Time { + return time.Now().UTC().Truncate(time.Microsecond) +} + +func formatTime(t time.Time) string { + return t.UTC().Format("2006-01-02T15:04:05.000000Z") +} + +func parseTime(s string) (time.Time, error) { + return time.Parse("2006-01-02T15:04:05.000000Z", s) +} + +type statusWriter struct { + http.ResponseWriter + status int +} + +func (w *statusWriter) WriteHeader(code int) { + w.status = code + w.ResponseWriter.WriteHeader(code) +} + +type ctxKey int + +const ( + reqIDKey ctxKey = 1 + routeKey ctxKey = 2 +) + +type routeHolder struct{ route string } + +func (a *App) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + reqID := newUUID() + w.Header().Set("X-Request-Id", reqID) + w.Header().Set("X-Instance-Id", instanceID) + sw := &statusWriter{ResponseWriter: w, status: 200} + health := strings.HasPrefix(r.URL.Path, "/health/") + rt := &routeHolder{route: "unmatched"} + ctx := context.WithValue(context.WithValue(r.Context(), reqIDKey, reqID), routeKey, rt) + var span trace.Span + if !health { + ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.HeaderCarrier(r.Header)) + ctx, span = tracer.Start(ctx, r.Method, trace.WithSpanKind(trace.SpanKindServer)) + span.SetAttributes(attribute.String("app.request_id", reqID)) + } + defer func() { + if rec := recover(); rec != nil { + a.writeErr(sw, reqID, 500, "internal_error", "internal error") + a.log.ErrorContext(ctx, "panic", "request_id", reqID, "err", fmt.Sprint(rec)) + } + elapsed := time.Since(start) + level := slog.LevelInfo + if health { + level = slog.LevelDebug + } else { + class := fmt.Sprintf("%dxx", sw.status/100) + attrs := metric.WithAttributes( + attribute.String("http.request.method", r.Method), + attribute.String("http.route", rt.route), + attribute.String("http.response.status_class", class), + ) + httpRequests.Add(ctx, 1, attrs) + httpDuration.Record(ctx, elapsed.Seconds(), attrs) + span.SetName(r.Method + " " + rt.route) + span.SetAttributes(attribute.String("http.request.method", r.Method), attribute.String("http.route", rt.route), + attribute.Int("http.response.status_code", sw.status)) + if sw.status >= 500 { + span.SetStatus(codes.Error, http.StatusText(sw.status)) + } + span.End() + } + a.log.Log(ctx, level, "request", + "method", r.Method, + "path", r.URL.Path, + "status", sw.status, + "duration_ms", elapsed.Milliseconds(), + "request_id", reqID, + ) + }() + next.ServeHTTP(sw, r.WithContext(ctx)) + }) +} + +func reqIDFrom(r *http.Request) string { + if v, ok := r.Context().Value(reqIDKey).(string); ok { + return v + } + return "" +} + +func (a *App) writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + _ = enc.Encode(v) +} + +func (a *App) writeErr(w http.ResponseWriter, reqID string, status int, code, msg string) { + a.writeJSON(w, status, errorBody{Error: apiError{Code: code, Message: msg, ReqID: reqID}}) +} + +func (a *App) writeUnauthorized(w http.ResponseWriter, r *http.Request, realm string) { + reqID := reqIDFrom(r) + if realm == "Basic" { + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + } else { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + } + a.writeErr(w, reqID, 401, "unauthorized", "unauthorized") +} + +func (a *App) Routes() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /health/live", a.handleLive) + mux.HandleFunc("GET /health/ready", a.handleReady) + mux.HandleFunc("POST /api/v1/users", a.handleCreateUser) + mux.HandleFunc("GET /api/v1/users/me", a.handleMe) + mux.HandleFunc("POST /api/v1/chats", a.handleCreateChat) + mux.HandleFunc("GET /api/v1/chats", a.handleListChats) + mux.HandleFunc("GET /api/v1/chats/{chat_id}", a.handleGetChat) + mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", a.handleCreateMessage) + mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", a.handleListMessages) + mux.HandleFunc("POST /api/v1/chats/{chat_id}/attachments", a.handleUploadAttachment) + mux.HandleFunc("GET /api/v1/attachments/{attachment_id}", a.handleGetAttachment) + mux.HandleFunc("GET /api/v1/attachments/{attachment_id}/content", a.handleDownloadAttachment) + mux.HandleFunc("POST /api/v1/auth/sessions", a.handleCreateSession) + mux.HandleFunc("DELETE /api/v1/auth/sessions/current", a.handleRevokeSession) + return a.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, pattern := mux.Handler(r) + if pattern == "" { + a.writeErr(w, reqIDFrom(r), 404, "not_found", "not found") + return + } + if rt, ok := r.Context().Value(routeKey).(*routeHolder); ok { + _, rt.route, _ = strings.Cut(pattern, " ") + } + mux.ServeHTTP(w, r) + })) +} + +func (a *App) handleLive(w http.ResponseWriter, r *http.Request) { + a.writeJSON(w, 200, map[string]string{"status": "ok"}) +} + +func (a *App) handleReady(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + if !a.ready.Load() { + a.writeErr(w, reqID, 503, "unavailable", "shutting down") + return + } + ctx, cancel := dbCtx(r.Context()) + defer cancel() + if err := a.db.Ping(ctx); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 200, map[string]string{"status": "ok"}) +} + +func (a *App) readJSON(w http.ResponseWriter, r *http.Request, dst any) bool { + reqID := reqIDFrom(r) + ct := r.Header.Get("Content-Type") + media := strings.ToLower(strings.TrimSpace(strings.Split(ct, ";")[0])) + if media != "application/json" { + a.writeErr(w, reqID, 415, "unsupported_media_type", "Content-Type must be application/json") + return false + } + r.Body = http.MaxBytesReader(w, r.Body, 1<<20) + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + if _, ok := err.(*http.MaxBytesError); ok { + a.writeErr(w, reqID, 413, "payload_too_large", "body too large") + return false + } + a.writeErr(w, reqID, 400, "invalid_request", "invalid json") + return false + } + var extra json.RawMessage + if err := dec.Decode(&extra); err != io.EOF { + a.writeErr(w, reqID, 400, "invalid_request", "invalid json") + return false + } + return true +} + +func isBlank(s string) bool { + return strings.TrimSpace(s) == "" +} + +func isUUID(s string) bool { + if len(s) != 36 { + return false + } + for i, c := range s { + switch i { + case 8, 13, 18, 23: + if c != '-' { + return false + } + default: + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { + return false + } + } + } + raw := strings.ReplaceAll(s, "-", "") + _, err := hex.DecodeString(raw) + return err == nil +} + +func isPgCode(err error, code string) bool { + var pgErr *pgconn.PgError + return errors.As(err, &pgErr) && pgErr.Code == code +} + +func userJSON(u *User) map[string]any { + return map[string]any{ + "id": u.ID, + "username": u.Username, + "display_name": u.DisplayName, + "created_at": formatTime(u.CreatedAt), + } +} + +func (a *App) authUser(w http.ResponseWriter, r *http.Request) (*User, string) { + token, ok := parseBearer(r.Header.Get("Authorization")) + if !ok { + a.writeUnauthorized(w, r, "Bearer") + return nil, "" + } + hash, hashHex := tokenHash(token) + + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var u User + var expiresAt time.Time + err := a.db.QueryRow(ctx, ` + select u.id::text, u.username, u.display_name, u.password_hash, u.created_at, s.expires_at + from sessions s join users u on u.id = s.user_id + where s.token_hash = $1 and s.expires_at > now()`, hash).Scan( + &u.ID, &u.Username, &u.DisplayName, &u.PasswordHash, &u.CreatedAt, &expiresAt) + if errors.Is(err, pgx.ErrNoRows) { + a.writeUnauthorized(w, r, "Bearer") + return nil, "" + } + if err != nil { + a.writeErr(w, reqIDFrom(r), 503, "unavailable", "database unavailable") + return nil, "" + } + return &u, hashHex +} + +func (a *App) handleCreateUser(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + var body struct { + Username *string `json:"username"` + DisplayName *string `json:"display_name"` + Password *string `json:"password"` + } + if !a.readJSON(w, r, &body) { + return + } + if body.Username == nil || body.DisplayName == nil || body.Password == nil { + a.writeErr(w, reqID, 400, "invalid_request", "username, display_name and password required") + return + } + username := *body.Username + display := *body.DisplayName + password := *body.Password + if !usernameRe.MatchString(username) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid username") + return + } + n := utf8.RuneCountInString(display) + if n < 1 || n > 100 || isBlank(display) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid display_name") + return + } + if !validPassword(password) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid password") + return + } + ph, err := hashPassword(password) + if err != nil { + a.writeErr(w, reqID, 500, "internal_error", "internal error") + return + } + id := newUUID() + createdAt := nowUTC() + ctx, cancel := dbCtx(r.Context()) + defer cancel() + _, err = a.db.Exec(ctx, ` + insert into users(id, username, display_name, password_hash, created_at) + values($1::uuid, $2, $3, $4, $5)`, id, username, display, ph, createdAt) + if isPgCode(err, "23505") { + a.writeErr(w, reqID, 409, "conflict", "username taken") + return + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 201, map[string]any{ + "id": id, + "username": username, + "display_name": display, + "created_at": formatTime(createdAt), + }) +} + +func (a *App) handleCreateSession(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + username, password, ok := parseBasic(r.Header.Get("Authorization")) + if !ok { + a.writeUnauthorized(w, r, "Basic") + return + } + if !usernameRe.MatchString(username) || !validPassword(password) { + a.writeUnauthorized(w, r, "Basic") + return + } + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var u User + err := a.db.QueryRow(ctx, ` + select id::text, username, display_name, password_hash, created_at + from users where username = $1`, username).Scan( + &u.ID, &u.Username, &u.DisplayName, &u.PasswordHash, &u.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + a.writeUnauthorized(w, r, "Basic") + return + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + match, err := checkPassword(u.PasswordHash, password) + if err != nil || !match { + a.writeUnauthorized(w, r, "Basic") + return + } + token, hash, _, err := newToken() + if err != nil { + a.writeErr(w, reqID, 500, "internal_error", "internal error") + return + } + createdAt := nowUTC() + expiresAt := createdAt.Add(a.sessionTTL) + _, err = a.db.Exec(ctx, ` + insert into sessions(token_hash, user_id, created_at, expires_at) + values($1, $2::uuid, $3, $4)`, hash, u.ID, createdAt, expiresAt) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 201, map[string]any{ + "token": token, + "token_type": "Bearer", + "expires_at": formatTime(expiresAt), + "user": userJSON(&u), + }) +} + +func (a *App) handleRevokeSession(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + token, ok := parseBearer(r.Header.Get("Authorization")) + if !ok { + a.writeUnauthorized(w, r, "Bearer") + return + } + hash, _ := tokenHash(token) + ctx, cancel := dbCtx(r.Context()) + defer cancel() + tag, err := a.db.Exec(ctx, ` + delete from sessions where token_hash = $1 and expires_at > now()`, hash) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if tag.RowsAffected() == 0 { + a.writeUnauthorized(w, r, "Bearer") + return + } + w.Header().Set("X-Request-Id", reqID) + w.WriteHeader(204) +} + +func (a *App) handleMe(w http.ResponseWriter, r *http.Request) { + u, _ := a.authUser(w, r) + if u == nil { + return + } + a.writeJSON(w, 200, userJSON(u)) +} + +func (a *App) handleCreateChat(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + var body struct { + Title *string `json:"title"` + MemberIDs *[]string `json:"member_ids"` + } + if !a.readJSON(w, r, &body) { + return + } + if body.Title == nil || body.MemberIDs == nil { + a.writeErr(w, reqID, 400, "invalid_request", "title and member_ids required") + return + } + title := *body.Title + n := utf8.RuneCountInString(title) + if n < 1 || n > 100 || isBlank(title) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid title") + return + } + members := *body.MemberIDs + if len(members) < 1 || len(members) > 100 { + a.writeErr(w, reqID, 400, "invalid_request", "invalid member_ids") + return + } + seen := make(map[string]struct{}, len(members)) + hasCaller := false + for _, id := range members { + if !isUUID(id) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid member id") + return + } + if _, dup := seen[id]; dup { + a.writeErr(w, reqID, 400, "invalid_request", "duplicate member") + return + } + seen[id] = struct{}{} + if id == u.ID { + hasCaller = true + } + } + if !hasCaller { + a.writeErr(w, reqID, 400, "invalid_request", "caller must be member") + return + } + + chatID := newUUID() + createdAt := nowUTC() + ctx, cancel := dbCtx(r.Context()) + defer cancel() + tx, err := a.db.Begin(ctx) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + defer tx.Rollback(ctx) + + _, err = tx.Exec(ctx, `insert into chats(id, title, created_at) values($1::uuid, $2, $3)`, + chatID, title, createdAt) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + for _, mid := range members { + _, err = tx.Exec(ctx, `insert into chat_members(chat_id, user_id) values($1::uuid, $2::uuid)`, + chatID, mid) + if isPgCode(err, "23503") { + a.writeErr(w, reqID, 400, "invalid_request", "unknown member") + return + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + } + if err := tx.Commit(ctx); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 201, map[string]any{ + "id": chatID, + "title": title, + "member_ids": members, + "created_at": formatTime(createdAt), + }) +} + +func encodeCursor(parts ...string) string { + return base64.RawURLEncoding.EncodeToString([]byte(strings.Join(parts, "|"))) +} + +func decodeCursor(s string) ([]string, error) { + b, err := base64.RawURLEncoding.DecodeString(s) + if err != nil { + return nil, err + } + return strings.Split(string(b), "|"), nil +} + +func parseLimit(r *http.Request) (int, error) { + q := r.URL.Query().Get("limit") + if q == "" { + return 50, nil + } + n, err := strconv.Atoi(q) + if err != nil || n < 1 || n > 100 { + return 0, fmt.Errorf("bad limit") + } + return n, nil +} + +func (a *App) loadChatMembers(ctx context.Context, chatID string) ([]string, error) { + ctx, cancel := dbCtx(ctx) + defer cancel() + rows, err := a.db.Query(ctx, ` + select user_id::text from chat_members where chat_id = $1::uuid order by user_id`, chatID) + if err != nil { + return nil, err + } + defer rows.Close() + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + return ids, rows.Err() +} + +func (a *App) isMember(ctx context.Context, chatID, userID string) (bool, error) { + ctx, cancel := dbCtx(ctx) + defer cancel() + var ok bool + err := a.db.QueryRow(ctx, ` + select exists( + select 1 from chat_members where chat_id = $1::uuid and user_id = $2::uuid + )`, chatID, userID).Scan(&ok) + return ok, err +} + +func (a *App) handleListChats(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + limit, err := parseLimit(r) + if err != nil { + a.writeErr(w, reqID, 400, "invalid_request", "invalid limit") + return + } + var afterTS *time.Time + var afterID string + if cur := r.URL.Query().Get("cursor"); cur != "" { + parts, err := decodeCursor(cur) + if err != nil || len(parts) != 3 { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + if parts[0] != u.ID { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + ts, err := parseTime(parts[1]) + if err != nil || !isUUID(parts[2]) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + afterTS = &ts + afterID = parts[2] + } + + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var rows pgx.Rows + if afterTS == nil { + rows, err = a.db.Query(ctx, ` + select c.id::text, c.title, c.created_at + from chats c + join chat_members m on m.chat_id = c.id + where m.user_id = $1::uuid + order by c.created_at, c.id + limit $2`, u.ID, limit+1) + } else { + rows, err = a.db.Query(ctx, ` + select c.id::text, c.title, c.created_at + from chats c + join chat_members m on m.chat_id = c.id + where m.user_id = $1::uuid + and (c.created_at, c.id) > ($2, $3::uuid) + order by c.created_at, c.id + limit $4`, u.ID, *afterTS, afterID, limit+1) + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + defer rows.Close() + + type row struct { + id, title string + created time.Time + } + var list []row + for rows.Next() { + var x row + if err := rows.Scan(&x.id, &x.title, &x.created); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + list = append(list, x) + } + if err := rows.Err(); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + hasMore := len(list) > limit + if hasMore { + list = list[:limit] + } + out := make([]map[string]any, 0, len(list)) + for _, c := range list { + members, err := a.loadChatMembers(r.Context(), c.id) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + out = append(out, map[string]any{ + "id": c.id, + "title": c.title, + "member_ids": members, + "created_at": formatTime(c.created), + }) + } + var next any = nil + if hasMore && len(list) > 0 { + last := list[len(list)-1] + next = encodeCursor(u.ID, formatTime(last.created), last.id) + } + a.writeJSON(w, 200, map[string]any{"items": out, "next_cursor": next}) +} + +func (a *App) handleGetChat(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + chatID := r.PathValue("chat_id") + if !isUUID(chatID) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id") + return + } + ok, err := a.isMember(r.Context(), chatID, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if !ok { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var title string + var createdAt time.Time + err = a.db.QueryRow(ctx, `select title, created_at from chats where id = $1::uuid`, chatID). + Scan(&title, &createdAt) + if errors.Is(err, pgx.ErrNoRows) { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + members, err := a.loadChatMembers(r.Context(), chatID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 200, map[string]any{ + "id": chatID, + "title": title, + "member_ids": members, + "created_at": formatTime(createdAt), + }) +} + +func (a *App) handleCreateMessage(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + chatID := r.PathValue("chat_id") + if !isUUID(chatID) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id") + return + } + key := r.Header.Get("Idempotency-Key") + if !idempotencyKeyRe.MatchString(key) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid Idempotency-Key") + return + } + ok, err := a.isMember(r.Context(), chatID, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if !ok { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + var body struct { + Text *string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + } + if !a.readJSON(w, r, &body) { + return + } + if body.Text == nil { + a.writeErr(w, reqID, 400, "invalid_request", "text required") + return + } + text := *body.Text + attachmentIDs := body.AttachmentIDs + if attachmentIDs == nil { + attachmentIDs = []string{} + } + if len(attachmentIDs) > 10 { + a.writeErr(w, reqID, 400, "invalid_request", "too many attachments") + return + } + seen := make(map[string]bool, len(attachmentIDs)) + for _, id := range attachmentIDs { + if !isUUID(id) || seen[id] { + a.writeErr(w, reqID, 400, "invalid_request", "invalid attachment_ids") + return + } + seen[id] = true + } + n := utf8.RuneCountInString(text) + if n > 4000 || (len(attachmentIDs) == 0 && (n < 1 || isBlank(text))) || (n > 0 && isBlank(text)) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid text") + return + } + if len(attachmentIDs) > 0 { + var found, failed int + cctx, ccancel := dbCtx(r.Context()) + err := a.db.QueryRow(cctx, ` + select count(*), count(*) filter (where status = 'failed') + from attachments where id = any($1::uuid[]) and chat_id = $2::uuid`, attachmentIDs, chatID).Scan(&found, &failed) + ccancel() + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if found != len(attachmentIDs) { + a.writeErr(w, reqID, 404, "not_found", "attachment not found") + return + } + if failed > 0 { + a.writeErr(w, reqID, 409, "conflict", "attachment processing failed") + return + } + } + hashInput := text + if len(attachmentIDs) > 0 { + hashInput += "\x00" + strings.Join(attachmentIDs, ",") + } + sum := sha256.Sum256([]byte(hashInput)) + bodyHash := hex.EncodeToString(sum[:]) + path := r.URL.Path + ctx, cancel := dbCtx(r.Context()) + defer cancel() + tx, err := a.db.Begin(ctx) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + defer tx.Rollback(ctx) + + if _, err := tx.Exec(ctx, ` + delete from idempotency_keys + where user_id=$1::uuid and method='POST' and path=$2 and key=$3 + and created_at < $4`, + u.ID, path, key, time.Now().Add(-a.idempotencyTTL)); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + msgID := newUUID() + createdAt := nowUTC() + resp := map[string]any{ + "id": msgID, + "chat_id": chatID, + "sender_id": u.ID, + "text": text, + "created_at": formatTime(createdAt), + "attachment_ids": attachmentIDs, + } + respJSON, _ := json.Marshal(resp) + tag, err := tx.Exec(ctx, ` + insert into idempotency_keys(user_id, method, path, key, body_hash, response, created_at) + values($1::uuid, 'POST', $2, $3, $4, $5, $6) + on conflict do nothing`, + u.ID, path, key, bodyHash, respJSON, createdAt) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if tag.RowsAffected() == 0 { + var storedHash string + var stored []byte + err := tx.QueryRow(ctx, ` + select body_hash, response from idempotency_keys + where user_id=$1::uuid and method='POST' and path=$2 and key=$3`, + u.ID, path, key).Scan(&storedHash, &stored) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if storedHash != bodyHash { + a.writeErr(w, reqID, 409, "conflict", "idempotency key reused with different body") + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(201) + _, _ = w.Write(append(stored, '\n')) + return + } + if _, err := tx.Exec(ctx, ` + insert into messages(id, chat_id, sender_id, text, created_at) + values($1::uuid, $2::uuid, $3::uuid, $4, $5)`, + msgID, chatID, u.ID, text, createdAt); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if len(attachmentIDs) > 0 { + if _, err := tx.Exec(ctx, ` + insert into message_attachments(message_id, attachment_id, position) + select $1::uuid, t.id, t.ord - 1 + from unnest($2::uuid[]) with ordinality as t(id, ord)`, + msgID, attachmentIDs); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + } + if err := tx.Commit(ctx); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 201, resp) +} + +func (a *App) handleListMessages(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + chatID := r.PathValue("chat_id") + if !isUUID(chatID) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id") + return + } + ok, err := a.isMember(r.Context(), chatID, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if !ok { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + limit, err := parseLimit(r) + if err != nil { + a.writeErr(w, reqID, 400, "invalid_request", "invalid limit") + return + } + var afterTS *time.Time + var afterID string + if cur := r.URL.Query().Get("cursor"); cur != "" { + parts, err := decodeCursor(cur) + if err != nil || len(parts) != 3 { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + if parts[0] != chatID { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + ts, err := parseTime(parts[1]) + if err != nil || !isUUID(parts[2]) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + afterTS = &ts + afterID = parts[2] + } + + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var rows pgx.Rows + if afterTS == nil { + rows, err = a.db.Query(ctx, ` + select id::text, chat_id::text, sender_id::text, text, created_at + from messages + where chat_id = $1::uuid + order by created_at, id + limit $2`, chatID, limit+1) + } else { + rows, err = a.db.Query(ctx, ` + select id::text, chat_id::text, sender_id::text, text, created_at + from messages + where chat_id = $1::uuid + and (created_at, id) > ($2, $3::uuid) + order by created_at, id + limit $4`, chatID, *afterTS, afterID, limit+1) + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + defer rows.Close() + + type row struct { + id, chatID, senderID, text string + created time.Time + } + var list []row + for rows.Next() { + var x row + if err := rows.Scan(&x.id, &x.chatID, &x.senderID, &x.text, &x.created); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + list = append(list, x) + } + if err := rows.Err(); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + hasMore := len(list) > limit + if hasMore { + list = list[:limit] + } + attachments := map[string][]string{} + if len(list) > 0 { + ids := make([]string, len(list)) + for i, m := range list { + ids[i] = m.id + } + arows, err := a.db.Query(ctx, ` + select message_id::text, attachment_id::text + from message_attachments + where message_id = any($1::uuid[]) + order by message_id, position`, ids) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + for arows.Next() { + var mid, aid string + if err := arows.Scan(&mid, &aid); err != nil { + arows.Close() + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + attachments[mid] = append(attachments[mid], aid) + } + arows.Close() + if err := arows.Err(); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + } + out := make([]map[string]any, 0, len(list)) + for _, m := range list { + ids := attachments[m.id] + if ids == nil { + ids = []string{} + } + out = append(out, map[string]any{ + "id": m.id, + "chat_id": m.chatID, + "sender_id": m.senderID, + "text": m.text, + "created_at": formatTime(m.created), + "attachment_ids": ids, + }) + } + var next any = nil + if hasMore && len(list) > 0 { + last := list[len(list)-1] + next = encodeCursor(chatID, formatTime(last.created), last.id) + } + a.writeJSON(w, 200, map[string]any{"items": out, "next_cursor": next}) +} + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} diff --git a/imaging.go b/imaging.go new file mode 100644 index 0000000..6ccc5f2 --- /dev/null +++ b/imaging.go @@ -0,0 +1,176 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + "image/draw" + "image/jpeg" + _ "image/png" + + "github.com/gen2brain/webp" + "github.com/rwcarlsen/goexif/exif" + xdraw "golang.org/x/image/draw" +) + +const ( + jpegQuality = 85 + webpQuality = 80 + webpMethod = 4 +) + +var errInvalidImage = errors.New("invalid image") + +type variantSpec struct { + name, contentType, ext string + box int + crop bool +} + +var variantSpecs = []variantSpec{ + {"thumbnail", "image/jpeg", "jpg", 256, false}, + {"crop", "image/jpeg", "jpg", 128, true}, + {"optimized", "image/webp", "webp", 1280, false}, +} + +type rendered struct { + spec variantSpec + data []byte + width, height int + sha256 string +} + +func renderVariants(ctx context.Context, data []byte, maxPixels int64) ([]rendered, error) { + cfg, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || cfg.Width < 1 || cfg.Height < 1 { + return nil, errInvalidImage + } + if int64(cfg.Width)*int64(cfg.Height) > maxPixels { + return nil, errInvalidImage + } + src, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return nil, errInvalidImage + } + orient := 1 + if format == "jpeg" { + orient = orientation(data) + } + src = shrink(src, 1280) + var out []rendered + for _, spec := range variantSpecs { + if err := ctx.Err(); err != nil { + return nil, err + } + img := render(src, cfg.Width, cfg.Height, orient, spec) + var buf bytes.Buffer + if spec.contentType == "image/webp" { + err = webp.Encode(&buf, img, webp.Options{Quality: webpQuality, Method: webpMethod}) + } else { + err = jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality}) + } + if err != nil { + return nil, err + } + sum := sha256.Sum256(buf.Bytes()) + b := img.Bounds() + out = append(out, rendered{spec: spec, data: buf.Bytes(), width: b.Dx(), height: b.Dy(), sha256: hex.EncodeToString(sum[:])}) + } + return out, nil +} + +func orientation(data []byte) int { + x, err := exif.Decode(bytes.NewReader(data)) + if err != nil { + return 1 + } + tag, err := x.Get(exif.Orientation) + if err != nil { + return 1 + } + v, err := tag.Int(0) + if err != nil || v < 1 || v > 8 { + return 1 + } + return v +} + +func fit(w, h, box int) (int, int) { + if w <= box && h <= box { + return w, h + } + if w >= h { + return box, max(1, h*box/w) + } + return max(1, w*box/h), box +} + +func shrink(src image.Image, limit int) image.Image { + for { + b := src.Bounds() + if max(b.Dx(), b.Dy())/2 < limit { + return src + } + dst := image.NewRGBA(image.Rect(0, 0, max(1, b.Dx()/2), max(1, b.Dy()/2))) + xdraw.ApproxBiLinear.Scale(dst, dst.Bounds(), src, b, xdraw.Src, nil) + src = dst + } +} + +func render(src image.Image, sw, sh, orient int, spec variantSpec) *image.RGBA { + b := src.Bounds() + rect := b + var tw, th int + if spec.crop { + side := min(b.Dx(), b.Dy()) + x0, y0 := b.Min.X+(b.Dx()-side)/2, b.Min.Y+(b.Dy()-side)/2 + rect = image.Rect(x0, y0, x0+side, y0+side) + tw, th = spec.box, spec.box + } else if orient >= 5 { + th, tw = fit(sh, sw, spec.box) + } else { + tw, th = fit(sw, sh, spec.box) + } + dst := image.NewRGBA(image.Rect(0, 0, tw, th)) + draw.Draw(dst, dst.Bounds(), image.White, image.Point{}, draw.Src) + xdraw.CatmullRom.Scale(dst, dst.Bounds(), src, rect, xdraw.Over, nil) + return orientImage(dst, orient) +} + +func orientImage(img *image.RGBA, o int) *image.RGBA { + if o <= 1 { + return img + } + w, h := img.Bounds().Dx(), img.Bounds().Dy() + dw, dh := w, h + if o >= 5 { + dw, dh = h, w + } + dst := image.NewRGBA(image.Rect(0, 0, dw, dh)) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + var nx, ny int + switch o { + case 2: + nx, ny = w-1-x, y + case 3: + nx, ny = w-1-x, h-1-y + case 4: + nx, ny = x, h-1-y + case 5: + nx, ny = y, x + case 6: + nx, ny = h-1-y, x + case 7: + nx, ny = h-1-y, w-1-x + case 8: + nx, ny = y, w-1-x + } + copy(dst.Pix[dst.PixOffset(nx, ny):dst.PixOffset(nx, ny)+4], img.Pix[img.PixOffset(x, y):img.PixOffset(x, y)+4]) + } + } + return dst +} diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..2715d9e --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,1317 @@ +{ + "uid": "messenger-observability", + "title": "Messenger observability", + "tags": [ + "messenger", + "lab6" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "SLI / SLO", + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Availability SLI (1 - 5xx share), selected range", + "description": "SLO: 99% of API requests are not 5xx over 30 days. Query: 1 - 5xx/all from http_server_requests_total.", + "gridPos": { + "x": 0, + "y": 1, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.99 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value" + }, + "targets": [ + { + "refId": "A", + "expr": "1 - (sum(increase(http_server_requests_total{http_response_status_class=\"5xx\"}[$__range])) or vector(0)) / sum(increase(http_server_requests_total[$__range]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "instant": true + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Ready within 10s SLI, selected range", + "description": "SLO: 95% of attachments reach ready within 10 s over 30 days. Bucket le=10 of job_time_to_ready_seconds.", + "gridPos": { + "x": 6, + "y": 1, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.95 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(increase(job_time_to_ready_seconds_bucket{le=\"10\"}[$__range])) / sum(increase(job_time_to_ready_seconds_count[$__range]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "instant": true + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Firing alerts", + "description": "Prometheus alert rules in infra/prometheus/rules/alerts.yml", + "gridPos": { + "x": 12, + "y": 1, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value" + }, + "targets": [ + { + "refId": "A", + "expr": "count(ALERTS{alertstate=\"firing\"}) or vector(0)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "instant": true + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Telemetry export errors, 5m", + "description": "Failed OTLP exports counted in the application SDK", + "gridPos": { + "x": 18, + "y": 1, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(increase(telemetry_export_errors_total[5m])) or vector(0)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "instant": true + } + ] + }, + { + "id": 6, + "type": "row", + "title": "API", + "gridPos": { + "x": 0, + "y": 5, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 7, + "type": "timeseries", + "title": "Requests per second by route", + "description": "Counter http_server_requests_total, rate over the dashboard interval", + "gridPos": { + "x": 0, + "y": 6, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (http_route) (rate(http_server_requests_total[$__rate_interval]))", + "legendFormat": "{{http_route}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "Error ratio (5xx / all)", + "description": "Numerator: 5xx responses, denominator: all API requests (health checks are not recorded). 4xx are client errors and are not counted.", + "gridPos": { + "x": 8, + "y": 6, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "thresholdsStyle": { + "mode": "line" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.05 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "(sum(rate(http_server_requests_total{http_response_status_class=\"5xx\"}[$__rate_interval])) or vector(0)) / sum(rate(http_server_requests_total[$__rate_interval]))", + "legendFormat": "5xx share", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 9, + "type": "timeseries", + "title": "Latency p50 / p95 / p99", + "description": "histogram_quantile over summed http_server_request_duration_seconds_bucket of all replicas", + "gridPos": { + "x": 16, + "y": 6, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 10, + "type": "row", + "title": "Jobs and queue", + "gridPos": { + "x": 0, + "y": 14, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 11, + "type": "timeseries", + "title": "Jobs per second by outcome", + "description": "jobs_processed_total: ready, retry, failed, skipped, lease_lost", + "gridPos": { + "x": 0, + "y": 15, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (outcome) (rate(jobs_processed_total[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "Job duration p50 / p95 / p99", + "description": "job_processing_duration_seconds_bucket, worker processing time of one attempt", + "gridPos": { + "x": 8, + "y": 15, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(job_processing_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(job_processing_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(job_processing_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 13, + "type": "timeseries", + "title": "Time to ready p50 / p95 / p99", + "description": "job_time_to_ready_seconds: from durable acceptance to ready commit", + "gridPos": { + "x": 16, + "y": 15, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(job_time_to_ready_seconds_bucket[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(job_time_to_ready_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(job_time_to_ready_seconds_bucket[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 14, + "type": "timeseries", + "title": "Queue depth", + "description": "Jobs in status queued (max across API replicas, they report the same PostgreSQL value)", + "gridPos": { + "x": 0, + "y": 23, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "max by (queue) (queue_depth)", + "legendFormat": "{{queue}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 15, + "type": "timeseries", + "title": "Oldest queued job age", + "description": "Seconds since the oldest available queued job became due; alert QueueLagGrowing fires above 20 s", + "gridPos": { + "x": 8, + "y": 23, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "thresholdsStyle": { + "mode": "line" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 20 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "max by (queue) (queue_oldest_job_age_seconds)", + "legendFormat": "{{queue}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 16, + "type": "timeseries", + "title": "Worker saturation", + "description": "worker_busy jobs in flight versus worker_capacity (WORKER_CONCURRENCY)", + "gridPos": { + "x": 16, + "y": 23, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum(worker_busy)", + "legendFormat": "busy", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum(worker_capacity)", + "legendFormat": "capacity", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 17, + "type": "row", + "title": "Saturation", + "gridPos": { + "x": 0, + "y": 31, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 18, + "type": "timeseries", + "title": "DB pool connections", + "description": "db_pool_connections by state, db_pool_max_connections limit", + "gridPos": { + "x": 0, + "y": 32, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (instance) (db_pool_connections{state=\"used\",job=\"messenger-api\"})", + "legendFormat": "used {{instance}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum(db_pool_connections{state=\"used\",job=\"messenger-worker\"})", + "legendFormat": "used worker", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "max(db_pool_max_connections)", + "legendFormat": "max", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 19, + "type": "timeseries", + "title": "DB pool acquire wait", + "description": "Time and count of acquires that had to wait for a free connection", + "gridPos": { + "x": 8, + "y": 32, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (job) (rate(db_pool_acquire_wait_seconds_total[$__rate_interval]))", + "legendFormat": "wait s/s {{job}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum by (job) (rate(db_pool_empty_acquires_total[$__rate_interval]))", + "legendFormat": "waited acquires/s {{job}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 20, + "type": "timeseries", + "title": "Process CPU", + "description": "CPU seconds per second of the application processes", + "gridPos": { + "x": 16, + "y": 32, + "w": 4, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (job) (rate(process_cpu_time_seconds_total[$__rate_interval]))", + "legendFormat": "{{job}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 21, + "type": "timeseries", + "title": "Process RSS", + "description": "Resident memory of the application processes", + "gridPos": { + "x": 20, + "y": 32, + "w": 4, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (job) (process_memory_usage_bytes)", + "legendFormat": "{{job}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 22, + "type": "row", + "title": "Telemetry pipeline", + "gridPos": { + "x": 0, + "y": 40, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 23, + "type": "timeseries", + "title": "Collector exporter queue", + "description": "Bounded sending queue of each Collector exporter (capacity 100 batches)", + "gridPos": { + "x": 0, + "y": 41, + "w": 8, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (exporter) (otelcol_exporter_queue_size)", + "legendFormat": "{{exporter}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 24, + "type": "timeseries", + "title": "Collector send failures", + "description": "Data the Collector could not deliver to a storage backend", + "gridPos": { + "x": 8, + "y": 41, + "w": 8, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum(rate(otelcol_exporter_send_failed_spans[$__rate_interval]))", + "legendFormat": "spans", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum(rate(otelcol_exporter_send_failed_metric_points[$__rate_interval]))", + "legendFormat": "metric points", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum(rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))", + "legendFormat": "log records", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 25, + "type": "timeseries", + "title": "Collector memory", + "description": "Collector RSS, bounded by memory_limiter and the container limit", + "gridPos": { + "x": 16, + "y": 41, + "w": 8, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "otelcol_process_memory_rss", + "legendFormat": "rss", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 26, + "type": "row", + "title": "Logs and traces", + "gridPos": { + "x": 0, + "y": 48, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 27, + "type": "logs", + "title": "Application logs (warn and error)", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "x": 0, + "y": 49, + "w": 12, + "h": 10 + }, + "options": { + "showTime": true, + "wrapLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{service_name=~\"messenger-.*\"} | detected_level=~\"warn|error\"" + } + ] + }, + { + "id": 28, + "type": "table", + "title": "Recent traces with errors", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "x": 12, + "y": 49, + "w": 12, + "h": 10 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "traceql", + "query": "{ status = error }", + "limit": 20, + "tableType": "traces" + } + ] + } + ], + "templating": { + "list": [] + }, + "annotations": { + "list": [] + } +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..2c7add2 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: messenger + type: file + disableDeletion: true + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..1e2a6af --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,34 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: trace_id + matcherType: label + matcherRegex: trace_id + url: "$${__value.raw}" + datasourceUid: tempo + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + filterByTraceID: true + customQuery: true + query: '{service_name=~"messenger-.*"} | trace_id="$${__trace.traceId}"' + serviceMap: + datasourceUid: prometheus diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..1f59a8d --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,47 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + log_level: warn + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +ingester: + wal: + enabled: true + dir: /loki/wal + flush_on_shutdown: true + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + retention_period: 168h + ingestion_rate_mb: 8 + ingestion_burst_size_mb: 16 + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..e844c36 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,77 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 160 + spike_limit_mib: 40 + batch: + timeout: 2s + send_batch_size: 512 + send_batch_max_size: 1024 + +exporters: + otlphttp/prometheus: + endpoint: http://prometheus:9090/api/v1/otlp + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 100 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 20s + otlphttp/loki: + endpoint: http://loki:3100/otlp + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 100 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 20s + otlp/tempo: + endpoint: tempo:4317 + timeout: 5s + tls: + insecure: true + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 100 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 20s + +service: + telemetry: + logs: + level: warn + metrics: + address: 0.0.0.0:8888 + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/tempo] diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..96a7622 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,19 @@ +global: + scrape_interval: 10s + evaluation_interval: 10s + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + +otlp: + promote_resource_attributes: + - service.version + - deployment.environment.name diff --git a/infra/prometheus/rules/alerts.yml b/infra/prometheus/rules/alerts.yml new file mode 100644 index 0000000..94c2c19 --- /dev/null +++ b/infra/prometheus/rules/alerts.yml @@ -0,0 +1,23 @@ +groups: + - name: messenger-slo + rules: + - alert: ApiHighErrorRate + expr: | + sum(rate(http_server_requests_total{http_response_status_class="5xx"}[1m])) + / sum(rate(http_server_requests_total[1m])) > 0.05 + for: 20s + labels: + severity: page + annotations: + summary: "API 5xx ratio above 5% over 1m" + description: "SLI: share of non-5xx business requests; SLO 99% over 30d. Fast-burn check on a 1m window." + runbook: "infra/runbook.md#apihigherrorrate" + - alert: QueueLagGrowing + expr: max(queue_oldest_job_age_seconds{queue="attachments.jobs"}) > 20 + for: 20s + labels: + severity: page + annotations: + summary: "Oldest queued job older than 20s" + description: "SLO: 95% of attachments reach ready within 10s over 30d. A job waiting more than 20s means the worker does not keep up." + runbook: "infra/runbook.md#queuelaggrowing" diff --git a/infra/runbook.md b/infra/runbook.md new file mode 100644 index 0000000..92dc600 --- /dev/null +++ b/infra/runbook.md @@ -0,0 +1,31 @@ +# Runbook + +Grafana: `http://127.0.0.1:3000` (пользователь `admin`, пароль в `secrets/grafana_admin_password`), dashboard «Messenger observability». Состояние alerts: `http://127.0.0.1:9090/alerts`. + +## ApiHighErrorRate + +Условие: доля ответов 5xx за 1 минуту больше 5% в течение 20 с. SLI: доля запросов API без 5xx, SLO 99% за 30 дней. + +1. Dashboard, «Error ratio» и «Requests per second by route»: какие маршруты дают 5xx. Запрос: `sum by (http_route) (rate(http_server_requests_total{http_response_status_class="5xx"}[1m]))`. +2. «Latency p50 / p95 / p99»: рост p99 до 3 с означает таймаут обращения к БД (`dbCtx` ограничивает запрос 3 с). +3. Tempo, запрос `{ resource.service.name = "messenger-api" && status = error }`: открыть trace. Если почти всё время server span занимает один дочерний `db ...` span со статусом error, причина в PostgreSQL, а не в обработчике. +4. «DB pool connections» и «DB pool acquire wait»: все соединения заняты или растёт ожидание подтверждает блокировку или медленную БД. +5. Loki: `{service_name="messenger-api"} | detected_level="error"`; по `trace_id` из лога открыть тот же trace. +6. PostgreSQL: `select pid, state, wait_event_type, left(query, 60) from pg_stat_activity where datname = 'messenger'` и `pg_locks` покажут блокирующую сессию. Завершить её (`pg_terminate_backend`) или устранить причину. +7. Alert разрешается сам, когда доля 5xx за минуту падает ниже порога. + +## QueueLagGrowing + +Условие: возраст самой старой доступной задачи в очереди `attachments.jobs` больше 20 с в течение 20 с. SLO: 95% вложений становятся `ready` за 10 с за 30 дней. + +1. Dashboard, «Queue depth» и «Oldest queued job age»: растут ли оба графика. +2. «Jobs per second by outcome» и «Worker saturation»: если `worker_busy` равен нулю и обработок нет, worker не работает. Если `worker_busy` равен `worker_capacity`, worker перегружен. +3. `docker compose --env-file .env ps worker` и Loki `{service_name="messenger-worker"}`: нет свежих логов означает остановленный процесс. +4. Tempo, `{ name = "attachments.jobs publish" }`: trace зависшей задачи содержит span публикации API и не содержит consumer span worker. +5. Если worker работает, но задачи падают: «Jobs per second by outcome» (`retry`, `failed`), Loki `{service_name="messenger-worker"} | detected_level=~"warn|error"`, trace попыток в одном trace, зависимость (S3, PostgreSQL) в error spans. +6. Запустить worker: `docker compose --env-file .env up -d worker`. Очередь разбирается, alert разрешается, когда возраст падает ниже порога. +7. Задачи в `failed`: `docker compose --env-file .env run --rm --no-deps -T worker failed`, повтор: `... worker replay `. + +## Недоступно хранилище телеметрии + +Приложение продолжает работать. Collector ограничивает память (`memory_limiter`, очередь экспортёра 100 батчей, повтор до 20 с), данные за время недоступности теряются. Потери: `otelcol_exporter_send_failed_*` на dashboard, «Collector send failures»; ошибки экспорта SDK: `telemetry_export_errors_total`. После запуска хранилища: `docker compose --env-file .env up -d prometheus loki tempo`. diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..c05cd63 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,27 @@ +server: + http_listen_port: 3200 + log_level: warn + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +compactor: + compaction: + block_retention: 168h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +usage_report: + reporting_enabled: false diff --git a/lb/haproxy.cfg b/lb/haproxy.cfg new file mode 100644 index 0000000..d63fed5 --- /dev/null +++ b/lb/haproxy.cfg @@ -0,0 +1,38 @@ +global + log stdout format raw local0 + maxconn 2000 + +resolvers docker + nameserver dns 127.0.0.11:53 + accepted_payload_size 8192 + hold valid 1s + hold nx 1s + hold obsolete 1s + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + +defaults + mode http + log global + option httplog + option redispatch 1 + retries 3 + retry-on conn-failure empty-response + timeout connect 1s + timeout client 20s + timeout server 15s + timeout http-request 10s + timeout queue 5s + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr none on-marked-down shutdown-sessions + server-template api 1-8 api:8080 diff --git a/main.go b/main.go new file mode 100644 index 0000000..a29bafd --- /dev/null +++ b/main.go @@ -0,0 +1,214 @@ +package main + +import ( + "context" + "encoding/json" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func main() { + log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})) + mode := "" + if len(os.Args) > 1 { + mode = os.Args[1] + } + if mode == "" || mode == "worker" { + service := "messenger-api" + if mode == "worker" { + service = "messenger-worker" + } + var flush func(context.Context) + log, flush = setupTelemetry(context.Background(), service) + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + flush(ctx) + }() + } + + host := envOr("APP_HOST", "0.0.0.0") + port := envOr("APP_PORT", "8080") + shutdownSec, _ := strconv.Atoi(envOr("SHUTDOWN_TIMEOUT_SECONDS", "5")) + if shutdownSec < 1 { + shutdownSec = 5 + } + ttlSec, _ := strconv.Atoi(envOr("SESSION_TTL_SECONDS", "3600")) + if ttlSec < 1 { + ttlSec = 3600 + } + + idemSec, _ := strconv.Atoi(envOr("IDEMPOTENCY_TTL_SECONDS", "86400")) + if idemSec < 1 { + idemSec = 86400 + } + drainSec, _ := strconv.Atoi(envOr("DRAIN_SECONDS", "3")) + + ctx := context.Background() + if len(os.Args) > 1 && os.Args[1] == "s3-init" { + if err := s3Init(ctx); err != nil { + log.Error("s3 init failed", "err", err.Error()) + os.Exit(1) + } + log.Info("bucket ready") + return + } + db, err := openDB(ctx) + if err != nil { + log.Error("database open failed", "err", err.Error()) + os.Exit(1) + } + defer db.Close() + + if len(os.Args) > 1 && os.Args[1] == "migrate" { + if err := migrate(ctx, db); err != nil { + log.Error("migrate failed", "err", err.Error()) + os.Exit(1) + } + log.Info("migrations applied") + return + } + + store, err := openStore() + if err != nil { + log.Error("s3 open failed", "err", err.Error()) + os.Exit(1) + } + + if len(os.Args) > 1 && (os.Args[1] == "cleanup" || os.Args[1] == "janitor") { + runCleanup(ctx, db, store, log, os.Args[1] == "janitor") + return + } + + maxBytes, _ := strconv.ParseInt(envOr("MAX_UPLOAD_BYTES", "10485760"), 10, 64) + maxPixels, _ := strconv.ParseInt(envOr("MAX_IMAGE_PIXELS", "40000000"), 10, 64) + maxAttempts := envInt("JOB_MAX_ATTEMPTS", 5) + + if len(os.Args) > 1 && (os.Args[1] == "replay" || os.Args[1] == "failed") { + out := json.NewEncoder(os.Stdout) + if os.Args[1] == "failed" { + items, err := listFailed(ctx, db) + if err != nil { + log.Error("failed jobs query failed", "err", err.Error()) + os.Exit(1) + } + for _, item := range items { + _ = out.Encode(item) + } + return + } + if len(os.Args) != 3 || !isUUID(os.Args[2]) { + log.Error("usage: replay ") + os.Exit(2) + } + result, err := replayJob(ctx, db, os.Args[2], maxAttempts) + if err != nil { + log.Error("replay failed", "err", err.Error()) + os.Exit(1) + } + _ = out.Encode(map[string]string{"attachment_id": os.Args[2], "result": result}) + return + } + + broker, err := openBroker() + if err != nil { + log.Error("broker open failed", "err", err.Error()) + os.Exit(1) + } + + if len(os.Args) > 1 && os.Args[1] == "worker" { + timeout := time.Duration(envInt("JOB_TIMEOUT_SECONDS", 60)) * time.Second + owner, _ := os.Hostname() + w := &Worker{ + db: db, store: store, broker: broker, log: log, + owner: owner + "-" + newUUID()[:8], + timeout: timeout, + lease: timeout + 15*time.Second, + backoffBase: time.Duration(envInt("JOB_BACKOFF_BASE_MS", 1000)) * time.Millisecond, + backoffMax: time.Duration(envInt("JOB_BACKOFF_MAX_MS", 30000)) * time.Millisecond, + maxBytes: maxBytes, + maxPixels: maxPixels, + fault: os.Getenv("FAULT_POINT"), + } + runWorker(db, store, broker, log, w, envInt("WORKER_CONCURRENCY", 2)) + return + } + + slots := envInt("UPLOAD_CONCURRENCY", 10) + app := NewApp(db, log, time.Duration(ttlSec)*time.Second, time.Duration(idemSec)*time.Second, store, + limits{maxBytes: maxBytes, maxPixels: maxPixels, uploadSlots: slots, maxAttempts: maxAttempts}) + registerQueueMetrics(db) + relayCtx, stopRelay := context.WithCancel(ctx) + defer stopRelay() + go runRelay(relayCtx, db, broker, log) + + addr := net.JoinHostPort(host, port) + srv := &http.Server{ + Addr: addr, + Handler: app.Routes(), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 15 * time.Second, + IdleTimeout: 30 * time.Second, + } + + go func() { + log.Info("listening", "addr", addr, "env", envOr("APP_ENV", "lab"), "session_ttl_s", ttlSec) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Error("server error", "err", err.Error()) + os.Exit(1) + } + }() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) + sig := <-stop + log.Info("shutdown started", "signal", sig.String()) + app.SetReady(false) + time.Sleep(time.Duration(drainSec) * time.Second) + + shCtx, cancel := context.WithTimeout(context.Background(), time.Duration(shutdownSec)*time.Second) + defer cancel() + if err := srv.Shutdown(shCtx); err != nil { + log.Error("shutdown error", "err", err.Error()) + os.Exit(1) + } + log.Info("shutdown finished") +} + +func runCleanup(ctx context.Context, db *pgxpool.Pool, store *Store, log *slog.Logger, loop bool) { + graceSec, _ := strconv.Atoi(envOr("CLEANUP_GRACE_SECONDS", "600")) + intervalSec, _ := strconv.Atoi(envOr("CLEANUP_INTERVAL_SECONDS", "300")) + retention := time.Duration(envInt("STAGING_RETENTION_SECONDS", 86400)) * time.Second + for { + removed, err := cleanupOrphans(ctx, db, store, time.Duration(graceSec)*time.Second, retention, log) + if err != nil { + log.Error("cleanup failed", "err", err.Error()) + if !loop { + os.Exit(1) + } + } else { + log.Info("cleanup finished", "removed", removed) + } + if !loop { + return + } + time.Sleep(time.Duration(intervalSec) * time.Second) + } +} + +func envInt(key string, def int) int { + v, err := strconv.Atoi(os.Getenv(key)) + if err != nil || v < 1 { + return def + } + return v +} diff --git a/migrate.go b/migrate.go new file mode 100644 index 0000000..d2b953f --- /dev/null +++ b/migrate.go @@ -0,0 +1,84 @@ +package main + +import ( + "context" + "embed" + "fmt" + "sort" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +//go:embed migrations/*.sql +var migrationFS embed.FS + +func migrate(ctx context.Context, pool *pgxpool.Pool) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + conn, err := pool.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + if _, err := conn.Exec(ctx, `select pg_advisory_lock(727001)`); err != nil { + return err + } + defer conn.Exec(context.Background(), `select pg_advisory_unlock(727001)`) + + _, err = conn.Exec(ctx, ` + create table if not exists schema_migrations ( + version text primary key, + applied_at timestamptz not null + )`) + if err != nil { + return fmt.Errorf("schema_migrations: %w", err) + } + + entries, err := migrationFS.ReadDir("migrations") + if err != nil { + return err + } + var names []string + for _, e := range entries { + if !e.IsDir() && strings.HasSuffix(e.Name(), ".sql") { + names = append(names, e.Name()) + } + } + sort.Strings(names) + + for _, name := range names { + var exists bool + err := conn.QueryRow(ctx, `select exists(select 1 from schema_migrations where version=$1)`, name).Scan(&exists) + if err != nil { + return err + } + if exists { + continue + } + + body, err := migrationFS.ReadFile("migrations/" + name) + if err != nil { + return err + } + + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + if _, err := tx.Exec(ctx, string(body)); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("migration %s: %w", name, err) + } + if _, err := tx.Exec(ctx, `insert into schema_migrations(version, applied_at) values($1, now())`, name); err != nil { + _ = tx.Rollback(ctx) + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + } + return nil +} diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 0000000..a50290e --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,38 @@ +create table users ( + id uuid primary key, + username text not null unique, + display_name text not null, + password_hash text not null, + created_at timestamptz not null +); + +create table chats ( + id uuid primary key, + title text not null, + created_at timestamptz not null +); + +create table chat_members ( + chat_id uuid not null references chats(id), + user_id uuid not null references users(id), + primary key (chat_id, user_id) +); + +create index chat_members_user_idx on chat_members (user_id); + +create table messages ( + id uuid primary key, + chat_id uuid not null references chats(id), + sender_id uuid not null references users(id), + text text not null, + created_at timestamptz not null +); + +create index messages_history_idx on messages (chat_id, created_at, id); + +create table sessions ( + token_hash bytea primary key, + user_id uuid not null references users(id) on delete cascade, + created_at timestamptz not null, + expires_at timestamptz not null +); diff --git a/migrations/0002_idempotency.sql b/migrations/0002_idempotency.sql new file mode 100644 index 0000000..0cfbc2d --- /dev/null +++ b/migrations/0002_idempotency.sql @@ -0,0 +1,10 @@ +create table idempotency_keys ( + user_id uuid not null references users(id) on delete cascade, + method text not null, + path text not null, + key text not null, + body_hash text not null, + response jsonb not null, + created_at timestamptz not null, + primary key (user_id, method, path, key) +); diff --git a/migrations/0003_attachments.sql b/migrations/0003_attachments.sql new file mode 100644 index 0000000..00d00fb --- /dev/null +++ b/migrations/0003_attachments.sql @@ -0,0 +1,22 @@ +create table attachments ( + id uuid primary key, + chat_id uuid not null references chats(id), + owner_id uuid not null references users(id), + filename text not null, + content_type text not null, + size_bytes bigint not null, + sha256 text not null, + object_key text not null unique, + created_at timestamptz not null +); + +create index attachments_chat_idx on attachments (chat_id); + +create table message_attachments ( + message_id uuid not null references messages(id), + attachment_id uuid not null references attachments(id), + position int not null, + primary key (message_id, attachment_id) +); + +create index message_attachments_attachment_idx on message_attachments (attachment_id); diff --git a/migrations/0004_jobs.sql b/migrations/0004_jobs.sql new file mode 100644 index 0000000..feddbf0 --- /dev/null +++ b/migrations/0004_jobs.sql @@ -0,0 +1,60 @@ +alter table attachments + add column status text not null default 'ready', + add column error_code text, + add column staging_key text, + add column updated_at timestamptz; + +update attachments set updated_at = created_at; + +alter table attachments + alter column status drop default, + alter column updated_at set not null, + alter column object_key drop not null; + +create table attachment_variants ( + attachment_id uuid not null references attachments(id), + pipeline_version int not null, + name text not null, + object_key text not null unique, + content_type text not null, + size_bytes bigint not null, + width int not null, + height int not null, + sha256 text not null, + primary key (attachment_id, pipeline_version, name) +); + +create table jobs ( + id uuid primary key, + attachment_id uuid not null unique references attachments(id), + pipeline_version int not null, + status text not null, + attempt int not null, + max_attempts int not null, + next_attempt_at timestamptz not null, + lease_owner text, + lease_until timestamptz, + last_error text, + created_at timestamptz not null, + updated_at timestamptz not null +); + +create index jobs_lease_idx on jobs (lease_until) where status = 'processing'; + +create table outbox ( + id bigserial primary key, + job_id uuid not null references jobs(id), + attempt int not null, + available_at timestamptz not null, + published_at timestamptz, + created_at timestamptz not null +); + +create index outbox_pending_idx on outbox (available_at) where published_at is null; + +create table job_replays ( + id bigserial primary key, + job_id uuid not null references jobs(id), + previous_error text, + replayed_at timestamptz not null +); diff --git a/migrations/0005_trace_context.sql b/migrations/0005_trace_context.sql new file mode 100644 index 0000000..ddda536 --- /dev/null +++ b/migrations/0005_trace_context.sql @@ -0,0 +1,5 @@ +alter table jobs + add column traceparent text, + add column tracestate text; + +create index jobs_queued_idx on jobs (next_attempt_at) where status = 'queued'; diff --git a/models.go b/models.go new file mode 100644 index 0000000..c6fd804 --- /dev/null +++ b/models.go @@ -0,0 +1,33 @@ +package main + +import "time" + +type User struct { + ID string + Username string + DisplayName string + PasswordHash string + CreatedAt time.Time +} + +type Chat struct { + ID string + Title string + MemberIDs []string + CreatedAt time.Time +} + +type Message struct { + ID string + ChatID string + SenderID string + Text string + CreatedAt time.Time +} + +type Session struct { + TokenHash []byte + UserID string + CreatedAt time.Time + ExpiresAt time.Time +} diff --git a/s3.go b/s3.go new file mode 100644 index 0000000..f1b381c --- /dev/null +++ b/s3.go @@ -0,0 +1,117 @@ +package main + +import ( + "context" + "io" + "net" + "net/http" + "net/url" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +type Store struct { + client *minio.Client + bucket string +} + +func openStore() (*Store, error) { + endpoint, err := url.Parse(envOr("S3_ENDPOINT", "http://s3:9000")) + if err != nil { + return nil, err + } + keyID, err := readSecret("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE") + if err != nil { + return nil, err + } + secret, err := readSecret("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE") + if err != nil { + return nil, err + } + transport := &http.Transport{ + DialContext: (&net.Dialer{Timeout: 2 * time.Second}).DialContext, + ResponseHeaderTimeout: 10 * time.Second, + MaxIdleConnsPerHost: 16, + IdleConnTimeout: 30 * time.Second, + } + client, err := minio.New(endpoint.Host, &minio.Options{ + Creds: credentials.NewStaticV4(keyID, secret, ""), + Secure: endpoint.Scheme == "https", + Region: envOr("S3_REGION", "us-east-1"), + Transport: transport, + MaxRetries: 1, + }) + if err != nil { + return nil, err + } + return &Store{client: client, bucket: envOr("S3_BUCKET", "messenger-attachments")}, nil +} + +func (s *Store) Put(ctx context.Context, key string, r io.Reader, size int64, contentType string) (err error) { + ctx, sp := childSpan(ctx, "s3 put", trace.SpanKindClient, attribute.String("s3.operation", "put")) + defer func() { endSpan(sp, err) }() + _, err = s.client.PutObject(ctx, s.bucket, key, r, size, minio.PutObjectOptions{ + ContentType: contentType, + PartSize: 5 << 20, + }) + return err +} + +func (s *Store) Get(ctx context.Context, key string) (_ *minio.Object, _ minio.ObjectInfo, err error) { + ctx, sp := childSpan(ctx, "s3 get", trace.SpanKindClient, attribute.String("s3.operation", "get")) + defer func() { endSpan(sp, err) }() + obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, minio.ObjectInfo{}, err + } + info, err := obj.Stat() + if err != nil { + obj.Close() + return nil, minio.ObjectInfo{}, err + } + return obj, info, nil +} + +func (s *Store) Remove(ctx context.Context, key string) (err error) { + ctx, sp := childSpan(ctx, "s3 remove", trace.SpanKindClient, attribute.String("s3.operation", "remove")) + defer func() { endSpan(sp, err) }() + return s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}) +} + +func (s *Store) List(ctx context.Context, prefix string) <-chan minio.ObjectInfo { + return s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) +} + +func isNoSuchKey(err error) bool { + return minio.ToErrorResponse(err).Code == "NoSuchKey" +} + +func s3Init(ctx context.Context) error { + s, err := openStore() + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + for { + exists, err := s.client.BucketExists(ctx, s.bucket) + if err == nil { + if exists { + return nil + } + err = s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: envOr("S3_REGION", "us-east-1")}) + if err == nil || minio.ToErrorResponse(err).Code == "BucketAlreadyOwnedByYou" { + return nil + } + } + select { + case <-ctx.Done(): + return err + case <-time.After(2 * time.Second): + } + } +} diff --git a/scripts/mksecrets.sh b/scripts/mksecrets.sh new file mode 100755 index 0000000..dfe9f53 --- /dev/null +++ b/scripts/mksecrets.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +mkdir -p "$ROOT/secrets" +cd "$ROOT/secrets" +gen() { + [ -f "$1" ] || openssl rand -hex "$2" | tr -d '\n' > "$1" +} +if [ ! -f postgres_password ]; then + openssl rand -base64 24 | tr -d '\n' > postgres_password +fi +PASS=$(cat postgres_password) +ENC=$(printf '%s' "$PASS" | python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.stdin.read(), safe=""))') +printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$ENC" > database_url +printf 'redis://valkey:6379/0\n' > cache_url +gen broker_password 24 +printf 'requirepass %s\n' "$(cat broker_password)" > broker.conf +printf 'redis://:%s@broker:6379/0\n' "$(cat broker_password)" > broker_url +gen grafana_admin_password 12 +gen s3_admin_access_key_id 8 +gen s3_admin_secret_access_key 20 +gen s3_access_key_id 8 +gen s3_secret_access_key 20 +BUCKET=$(grep -E '^S3_BUCKET=' "$ROOT/.env" 2>/dev/null | cut -d= -f2 || true) +BUCKET=${BUCKET:-messenger-attachments} +printf '{"identities":[{"name":"admin","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write","List","Tagging"]},{"name":"api","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Read:%s","Write:%s","List:%s"]}]}\n' \ + "$(cat s3_admin_access_key_id)" "$(cat s3_admin_secret_access_key)" "$(cat s3_access_key_id)" "$(cat s3_secret_access_key)" "$BUCKET" "$BUCKET" "$BUCKET" > s3_config.json +chmod 644 ./* +echo "secrets ready in $ROOT/secrets" diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..25b2375 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +docker compose --env-file .env restart postgres valkey broker s3 prometheus loki tempo +docker compose --env-file .env up -d --wait postgres valkey broker s3 prometheus loki tempo +docker compose --env-file .env restart otel-collector grafana api lb janitor worker diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 0000000..9cee552 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +dc="docker compose --env-file .env" +old=$($dc ps -q api) +n=$(echo "$old" | wc -w) +$dc build migrate +$dc up -d --no-deps migrate janitor worker +$dc up -d --no-deps --no-recreate --scale api=$((n * 2)) api +$dc up -d --wait --no-deps --no-recreate --scale api=$((n * 2)) api +for c in $old; do + docker stop -t 10 "$c" >/dev/null + docker rm "$c" >/dev/null +done +$dc up -d --wait --no-deps --no-recreate --scale api=$n api diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..1283ec1 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?}" +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +id=$(docker compose --env-file .env ps -q api | while read -r c; do + case "$c" in "$TARGET_INSTANCE_ID"*) echo "$c";; esac +done) +[ -n "$id" ] || exit 1 +docker stop -t 10 "$id" >/dev/null diff --git a/telemetry.go b/telemetry.go new file mode 100644 index 0000000..0bb62e8 --- /dev/null +++ b/telemetry.go @@ -0,0 +1,352 @@ +package main + +import ( + "context" + "log/slog" + "os" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/contrib/bridges/otelslog" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/propagation" + sdklog "go.opentelemetry.io/otel/sdk/log" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" +) + +const appVersion = "1.6.0" + +var ( + tracer = otel.Tracer("messenger") + meter = otel.Meter("messenger") + + httpRequests = mustCounter("http.server.requests", "{request}", "HTTP requests by method, route template and status class") + httpDuration = mustHistogram("http.server.request.duration", "s", "HTTP server request duration", + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10) + jobsProcessed = mustCounter("jobs.processed", "{job}", "Processed job attempts by outcome") + jobDuration = mustHistogram("job.processing.duration", "s", "Job processing duration", + 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60) + jobTimeToReady = mustHistogram("job.time_to_ready", "s", "Time from durable acceptance to ready", + 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300) + + exportErrors atomic.Int64 +) + +type exportErrorHandler struct{} + +func (exportErrorHandler) Handle(error) { exportErrors.Add(1) } + +func mustCounter(name, unit, desc string) metric.Int64Counter { + c, err := meter.Int64Counter(name, metric.WithUnit(unit), metric.WithDescription(desc)) + if err != nil { + panic(err) + } + return c +} + +func mustHistogram(name, unit, desc string, buckets ...float64) metric.Float64Histogram { + h, err := meter.Float64Histogram(name, metric.WithUnit(unit), metric.WithDescription(desc), + metric.WithExplicitBucketBoundaries(buckets...)) + if err != nil { + panic(err) + } + return h +} + +type fanoutHandler struct { + stdout, remote slog.Handler + service string +} + +func (h fanoutHandler) Enabled(ctx context.Context, l slog.Level) bool { + return h.stdout.Enabled(ctx, l) +} + +func (h fanoutHandler) Handle(ctx context.Context, r slog.Record) error { + local := r.Clone() + local.AddAttrs(slog.String("service.name", h.service), slog.String("event", r.Message)) + if sc := trace.SpanContextFromContext(ctx); sc.IsValid() { + local.AddAttrs(slog.String("trace_id", sc.TraceID().String()), slog.String("span_id", sc.SpanID().String())) + } + err := h.stdout.Handle(ctx, local) + remote := r.Clone() + remote.AddAttrs(slog.String("event", r.Message)) + _ = h.remote.Handle(ctx, remote) + return err +} + +func (h fanoutHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return fanoutHandler{stdout: h.stdout.WithAttrs(attrs), remote: h.remote.WithAttrs(attrs), service: h.service} +} + +func (h fanoutHandler) WithGroup(name string) slog.Handler { + return fanoutHandler{stdout: h.stdout.WithGroup(name), remote: h.remote.WithGroup(name), service: h.service} +} + +func setupTelemetry(ctx context.Context, defaultService string) (*slog.Logger, func(context.Context)) { + service := envOr("OTEL_SERVICE_NAME", defaultService) + res, err := resource.New(ctx, + resource.WithAttributes( + attribute.String("service.name", service), + attribute.String("service.version", appVersion), + attribute.String("service.instance.id", instanceID), + attribute.String("deployment.environment.name", envOr("APP_ENV", "lab")), + ), + resource.WithFromEnv(), + ) + if err != nil { + panic(err) + } + otel.SetErrorHandler(exportErrorHandler{}) + otel.SetTextMapPropagator(propagation.TraceContext{}) + + traceExp, err := otlptracehttp.New(ctx, + otlptracehttp.WithTimeout(3*time.Second), + otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: true, InitialInterval: time.Second, MaxInterval: 4 * time.Second, MaxElapsedTime: 8 * time.Second})) + if err != nil { + panic(err) + } + tp := sdktrace.NewTracerProvider( + sdktrace.WithResource(res), + sdktrace.WithBatcher(traceExp, + sdktrace.WithBatchTimeout(time.Second), + sdktrace.WithMaxQueueSize(2048), + sdktrace.WithMaxExportBatchSize(512), + sdktrace.WithExportTimeout(5*time.Second)), + ) + otel.SetTracerProvider(tp) + + metricExp, err := otlpmetrichttp.New(ctx, + otlpmetrichttp.WithTimeout(3*time.Second), + otlpmetrichttp.WithRetry(otlpmetrichttp.RetryConfig{Enabled: true, InitialInterval: time.Second, MaxInterval: 4 * time.Second, MaxElapsedTime: 8 * time.Second})) + if err != nil { + panic(err) + } + mp := sdkmetric.NewMeterProvider( + sdkmetric.WithResource(res), + sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExp, sdkmetric.WithInterval(10*time.Second), sdkmetric.WithTimeout(5*time.Second))), + ) + otel.SetMeterProvider(mp) + + logExp, err := otlploghttp.New(ctx, + otlploghttp.WithTimeout(3*time.Second), + otlploghttp.WithRetry(otlploghttp.RetryConfig{Enabled: true, InitialInterval: time.Second, MaxInterval: 4 * time.Second, MaxElapsedTime: 8 * time.Second})) + if err != nil { + panic(err) + } + lp := sdklog.NewLoggerProvider( + sdklog.WithResource(res), + sdklog.WithProcessor(sdklog.NewBatchProcessor(logExp, + sdklog.WithMaxQueueSize(2048), + sdklog.WithExportInterval(time.Second), + sdklog.WithExportTimeout(5*time.Second))), + ) + + registerProcessMetrics() + log := slog.New(fanoutHandler{ + stdout: slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}), + remote: otelslog.NewHandler("messenger", otelslog.WithLoggerProvider(lp)), + service: service, + }) + shutdown := func(ctx context.Context) { + _ = tp.Shutdown(ctx) + _ = mp.Shutdown(ctx) + _ = lp.Shutdown(ctx) + } + return log, shutdown +} + +func observableGauge(name, unit, desc string, cb func(context.Context, metric.Int64Observer) error) { + _, err := meter.Int64ObservableGauge(name, metric.WithUnit(unit), metric.WithDescription(desc), metric.WithInt64Callback(cb)) + if err != nil { + panic(err) + } +} + +func observableCounter(name, unit, desc string, cb func(context.Context, metric.Float64Observer) error) { + _, err := meter.Float64ObservableCounter(name, metric.WithUnit(unit), metric.WithDescription(desc), metric.WithFloat64Callback(cb)) + if err != nil { + panic(err) + } +} + +func registerProcessMetrics() { + observableCounter("process.cpu.time", "s", "Process CPU time", func(_ context.Context, o metric.Float64Observer) error { + var ru syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &ru); err != nil { + return nil + } + sec := func(t syscall.Timeval) float64 { return float64(t.Sec) + float64(t.Usec)/1e6 } + o.Observe(sec(ru.Utime) + sec(ru.Stime)) + return nil + }) + observableGauge("process.memory.usage", "By", "Resident set size", func(_ context.Context, o metric.Int64Observer) error { + data, err := os.ReadFile("/proc/self/status") + if err != nil { + return nil + } + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "VmRSS:") { + f := strings.Fields(line) + if len(f) >= 2 { + var kb int64 + for _, c := range f[1] { + kb = kb*10 + int64(c-'0') + } + o.Observe(kb * 1024) + } + } + } + return nil + }) + observableCounter("telemetry.export.errors", "{error}", "Failed telemetry exports", func(_ context.Context, o metric.Float64Observer) error { + o.Observe(float64(exportErrors.Load())) + return nil + }) +} + +func registerPoolMetrics(db *pgxpool.Pool) { + observableGauge("db.pool.connections", "{connection}", "DB pool connections by state", func(_ context.Context, o metric.Int64Observer) error { + s := db.Stat() + o.Observe(int64(s.AcquiredConns()), metric.WithAttributes(attribute.String("state", "used"))) + o.Observe(int64(s.IdleConns()), metric.WithAttributes(attribute.String("state", "idle"))) + return nil + }) + observableGauge("db.pool.max_connections", "{connection}", "DB pool size limit", func(_ context.Context, o metric.Int64Observer) error { + o.Observe(int64(db.Stat().MaxConns())) + return nil + }) + observableCounter("db.pool.empty_acquires", "{acquire}", "Acquires that had to wait for a connection", func(_ context.Context, o metric.Float64Observer) error { + o.Observe(float64(db.Stat().EmptyAcquireCount())) + return nil + }) + observableCounter("db.pool.acquire.wait", "s", "Total time spent acquiring connections", func(_ context.Context, o metric.Float64Observer) error { + o.Observe(db.Stat().AcquireDuration().Seconds()) + return nil + }) +} + +func registerQueueMetrics(db *pgxpool.Pool) { + var mu sync.Mutex + var depth int64 + var age float64 + refresh := func(ctx context.Context) { + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + var d int64 + var a float64 + err := db.QueryRow(ctx, ` + select count(*), + coalesce(extract(epoch from now() - min(next_attempt_at) filter (where next_attempt_at <= now())), 0) + from jobs where status = 'queued'`).Scan(&d, &a) + if err != nil { + return + } + mu.Lock() + depth, age = d, a + mu.Unlock() + } + attrs := metric.WithAttributes(attribute.String("queue", jobStream)) + observableGauge("queue.depth", "{job}", "Queued jobs", func(ctx context.Context, o metric.Int64Observer) error { + refresh(ctx) + mu.Lock() + defer mu.Unlock() + o.Observe(depth, attrs) + return nil + }) + _, err := meter.Float64ObservableGauge("queue.oldest_job_age", metric.WithUnit("s"), metric.WithDescription("Age of the oldest available queued job"), + metric.WithFloat64Callback(func(_ context.Context, o metric.Float64Observer) error { + mu.Lock() + defer mu.Unlock() + o.Observe(age, attrs) + return nil + })) + if err != nil { + panic(err) + } +} + +func registerWorkerMetrics(w *Worker, concurrency int) { + observableGauge("worker.busy", "{slot}", "Jobs being processed", func(_ context.Context, o metric.Int64Observer) error { + o.Observe(w.busy.Load()) + return nil + }) + observableGauge("worker.capacity", "{slot}", "Configured worker concurrency", func(_ context.Context, o metric.Int64Observer) error { + o.Observe(int64(concurrency)) + return nil + }) +} + +func childSpan(ctx context.Context, name string, kind trace.SpanKind, attrs ...attribute.KeyValue) (context.Context, trace.Span) { + if !trace.SpanContextFromContext(ctx).IsValid() { + return ctx, trace.SpanFromContext(ctx) + } + return tracer.Start(ctx, name, trace.WithSpanKind(kind), trace.WithAttributes(attrs...)) +} + +func endSpan(sp trace.Span, err error) { + if err != nil { + sp.RecordError(err) + sp.SetStatus(codes.Error, err.Error()) + } + sp.End() +} + +type traceCarrier struct{ traceparent, tracestate string } + +func injectTrace(ctx context.Context) traceCarrier { + m := propagation.MapCarrier{} + otel.GetTextMapPropagator().Inject(ctx, m) + return traceCarrier{traceparent: m.Get("traceparent"), tracestate: m.Get("tracestate")} +} + +func extractTrace(ctx context.Context, c traceCarrier) context.Context { + if c.traceparent == "" { + return ctx + } + m := propagation.MapCarrier{"traceparent": c.traceparent} + if c.tracestate != "" { + m["tracestate"] = c.tracestate + } + return otel.GetTextMapPropagator().Extract(ctx, m) +} + +type dbTracer struct{} + +type dbSpanKey struct{} + +func (dbTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context { + if !trace.SpanContextFromContext(ctx).IsValid() { + return ctx + } + op := "query" + if f := strings.Fields(data.SQL); len(f) > 0 { + op = strings.ToUpper(f[0]) + } + ctx, sp := tracer.Start(ctx, "db "+op, trace.WithSpanKind(trace.SpanKindClient), trace.WithAttributes( + attribute.String("db.system.name", "postgresql"), + attribute.String("db.operation.name", op), + attribute.String("db.query.text", data.SQL), + )) + return context.WithValue(ctx, dbSpanKey{}, sp) +} + +func (dbTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + if sp, ok := ctx.Value(dbSpanKey{}).(trace.Span); ok { + endSpan(sp, data.Err) + } +} diff --git a/tests/attachments.py b/tests/attachments.py new file mode 100644 index 0000000..871480c --- /dev/null +++ b/tests/attachments.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +import hashlib +import uuid +from pathlib import Path +from client import Client, arguments, check, run +from image_gen import png, png_header_only +from stack import wait_status + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() +MAX_BYTES = 10 * 1024 * 1024 + + +def upload(c, actor, chat, data, expected, content_type='image/png', filename='file.png'): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, expected, + actor=actor, raw=data, headers={'Content-Type': content_type}, + schema='Attachment' if expected == 202 else None) + + +def post_message(c, actor, chat, ids, expected, text=''): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', expected, actor=actor, + body={'text': text, 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message' if expected == 201 else None) + + +def main(): + args = arguments('Attachment validation and access checks') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'second', 'member_ids': [alice['id'], bob['id']]}, schema='Chat')['json'] + eve_chat = c.call('POST', '/api/v1/chats', 201, actor=eve, body={'title': 'eve', 'member_ids': [eve['id'], bob['id']]}, schema='Chat')['json'] + + upload(c, alice, chat, b'x' * (MAX_BYTES + 1), 413) + upload(c, alice, chat, SAMPLE + b'\0' * MAX_BYTES, 413) + upload(c, alice, chat, png_header_only(8000, 8000), 413) + upload(c, alice, chat, SAMPLE[:len(SAMPLE) // 2], 415) + upload(c, alice, chat, SAMPLE[:-1], 415) + upload(c, alice, chat, b'\x89PNG\r\n\x1a\n' + b'garbage' * 100, 415) + upload(c, alice, chat, b'', 415) + upload(c, alice, chat, b'GIF89a' + b'\0' * 64, 415) + upload(c, alice, chat, SAMPLE, 415, content_type='image/jpeg') + upload(c, alice, chat, SAMPLE, 415, content_type='text/plain') + upload(c, alice, chat, SAMPLE, 415, content_type='application/octet-stream') + + for name in ('..%2Fx.png', 'a%5Cb.png', 'a%00b.png', 'a%0Ab.png', 'a' * 256, ''): + upload(c, alice, chat, SAMPLE, 400, filename=name) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments', 400, actor=alice, raw=SAMPLE, headers={'Content-Type': 'image/png'}) + upload(c, eve, chat, SAMPLE, 404) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=a.png', 401, raw=SAMPLE, headers={'Content-Type': 'image/png'}) + + ok = upload(c, alice, chat, SAMPLE, 202)['json'] + check(ok['sha256'] == hashlib.sha256(SAMPLE).hexdigest(), 'checksum mismatch') + check(ok['filename'] == 'file.png' and ok['status'] == 'queued' and ok['variants'] == [], 'attachment fields') + unicode_name = upload(c, alice, chat, SAMPLE, 202, filename='%D1%84%D0%BE%D1%82%D0%BE.png')['json'] + check(unicode_name['filename'] == 'фото.png', 'unicode filename changed') + + post_message(c, alice, other, [ok['id']], 404) + post_message(c, alice, chat, [str(uuid.uuid4())], 404) + post_message(c, alice, chat, [ok['id'], ok['id']], 400) + post_message(c, alice, chat, ['not-a-uuid'], 400) + post_message(c, alice, chat, [str(uuid.uuid4()) for _ in range(11)], 400) + post_message(c, alice, chat, [], 400) + post_message(c, eve, chat, [ok['id']], 404) + post_message(c, eve, eve_chat, [ok['id']], 404) + + shared = post_message(c, bob, chat, [ok['id'], unicode_name['id']], 201, text='from bob')['json'] + check(shared['attachment_ids'] == [ok['id'], unicode_name['id']], 'attachment order changed') + history = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=alice, schema='MessagePage')['json']['items'] + check(any(m['id'] == shared['id'] and m['attachment_ids'] == shared['attachment_ids'] for m in history), 'history lost attachment_ids') + + for suffix in ('', '/content'): + c.call('GET', '/api/v1/attachments/' + ok['id'] + suffix, 404, actor=eve) + c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()) + suffix, 404, actor=alice) + c.call('GET', '/api/v1/attachments/bad' + suffix, 400, actor=alice) + c.call('GET', '/api/v1/attachments/' + ok['id'] + suffix, 401) + c.call('GET', '/api/v1/attachments/' + ok['id'] + '/content?variant=bogus', 400, actor=bob) + wait_status(c, bob, ok['id'], ('ready',), 60) + download = c.call('GET', '/api/v1/attachments/' + ok['id'] + '/content', actor=bob) + check(download['bytes'] == SAMPLE, 'download differs') + + big = png(1800, 1800, noise=True) + check(len(big) < MAX_BYTES, 'generated image exceeds limit') + large = upload(c, alice, chat, big, 202)['json'] + wait_status(c, bob, large['id'], ('ready',), 60) + check(c.call('GET', '/api/v1/attachments/' + large['id'] + '/content', actor=bob)['bytes'] == big, 'large download differs') + print('PASS: attachment validation, limits and access control') + + +if __name__ == '__main__': + run(main) diff --git a/tests/dashboard.py b/tests/dashboard.py new file mode 100644 index 0000000..3f2d6f9 --- /dev/null +++ b/tests/dashboard.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +import base64 +import json +import urllib.parse +import urllib.request +from pathlib import Path +from client import ROOT, arguments, check, run + +GRAFANA = 'http://127.0.0.1:3000' + + +def call(method, path, body=None): + password = (ROOT / 'secrets/grafana_admin_password').read_text().strip() + request = urllib.request.Request(GRAFANA + path, data=json.dumps(body).encode() if body is not None else None, method=method, + headers={'Content-Type': 'application/json', + 'Authorization': 'Basic ' + base64.b64encode(('admin:' + password).encode()).decode()}) + with urllib.request.urlopen(request, timeout=30) as response: + return json.loads(response.read()) + + +def main(): + arguments('Provisioned datasources and dashboard answer their queries through Grafana') + sources = {d['uid']: d['type'] for d in call('GET', '/api/datasources')} + check(sources == {'prometheus': 'prometheus', 'loki': 'loki', 'tempo': 'tempo'}, 'datasources not provisioned: %s' % sources) + found = call('GET', '/api/search?query=Messenger') + check([d['uid'] for d in found] == ['messenger-observability'], 'dashboard not provisioned') + dashboard = call('GET', '/api/dashboards/uid/messenger-observability')['dashboard'] + on_disk = json.loads((ROOT / 'infra/grafana/dashboards/messenger.json').read_text()) + check(len(dashboard['panels']) == len(on_disk['panels']), 'dashboard differs from the file in Git') + checked = 0 + for panel in dashboard['panels']: + for target in panel.get('targets', []): + if panel['datasource']['type'] == 'tempo': + found = call('GET', '/api/datasources/proxy/uid/tempo/api/search?q=' + urllib.parse.quote(target['query']) + '&limit=5') + check('traces' in found, 'panel "%s" tempo search failed' % panel['title']) + checked += 1 + continue + query = dict(target, intervalMs=15000, maxDataPoints=500) + result = call('POST', '/api/ds/query', {'queries': [query], 'from': 'now-30m', 'to': 'now'})['results'][target['refId']] + check('error' not in result and result.get('status', 200) == 200, 'panel "%s" query failed: %s' % (panel['title'], result.get('error'))) + checked += 1 + print('grafana: 3 datasources and dashboard "%s" provisioned from Git, %d panels, %d queries executed without errors' + % (dashboard['title'], len(dashboard['panels']), checked)) + print('PASS: provisioning and dashboard queries') + + +if __name__ == '__main__': + run(main) diff --git a/tests/faults.py b/tests/faults.py new file mode 100755 index 0000000..d4edb60 --- /dev/null +++ b/tests/faults.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +import json +import re +import time +import uuid +from client import Client, Failure, arguments, check, run +from stack import (SAMPLE, broker, compose, job_row, logs, now_iso, psql, restore_api, restore_worker, upload, + variant_count, wait_api, wait_log, wait_status) + + +def job_id_of(att_id): + return psql("select id from jobs where attachment_id='%s'" % att_id) + + +def worker_with(fault, **extra): + compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env=dict({'WORKER_FAULT_POINT': fault}, **extra)) + + +def wait_s3(timeout=90): + deadline = time.monotonic() + timeout + while '(healthy)' not in compose('ps', 's3', '--format', '{{.Status}}').stdout: + check(time.monotonic() < deadline, 's3 did not recover') + time.sleep(1) + + +def ops(*args): + out = compose('run', '--rm', '--no-deps', '-T', 'worker', *args).stdout + return [json.loads(line) for line in out.splitlines() if line.startswith('{')] + + +def crash_after_commit(c, alice, bob, chat): + since = now_iso() + compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': 'after_commit'}) + compose('restart', 'lb') + wait_api(c) + name = 'crash-' + uuid.uuid4().hex[:12] + '.png' + answered = None + try: + answered = upload(c, alice, chat, filename=name) + except Failure: + pass + check(answered is None, 'api answered 202 despite injected crash') + wait_log('api', since, '"point":"after_commit"', 20) + att_id = psql("select id from attachments where filename='%s'" % name) + check(att_id, 'committed attachment not found') + restore_api(c) + ready = wait_status(c, bob, att_id, ('ready', 'failed'), 60) + check(ready['status'] == 'ready' and job_row(att_id)[:2] == ('ready', 1), 'outbox did not recover the job') + published = psql("select count(*) from outbox where job_id='%s' and published_at is not null" % job_id_of(att_id)) + print('1) api crashed after COMMIT before 202/publish: attachment=%s recovered by outbox relay (published rows=%s) -> ready attempt=1' + % (att_id, published)) + + +def crash_after_store(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + since = now_iso() + worker_with('after_store', JOB_TIMEOUT_SECONDS='10') + wait_log('worker', since, '"point":"after_store"', 60) + status, attempt, _ = job_row(att['id']) + check(status == 'processing' and attempt == 1, 'job must stay processing under expired-to-be lease') + since = now_iso() + worker_with('', JOB_TIMEOUT_SECONDS='10') + wait_log('worker', since, '"lease expired","job_id":"%s"' % job_id_of(att['id']), 90) + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + status, attempt, _ = job_row(att['id']) + check(ready['status'] == 'ready' and status == 'ready' and attempt >= 2, 'job did not finish after lease expiry') + check(variant_count(att['id']) == 3 and len(ready['variants']) == 3, 'retry duplicated variants') + thumb = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0] + body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)['bytes'] + check(len(body) == thumb['size_bytes'], 'thumbnail differs from metadata') + restore_worker() + print('2) worker crashed after S3 write before COMMIT: lease expired, reaper requeued, attempt=%d ready, variants=3' % attempt) + + +def crash_before_ack(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + job_id = job_id_of(att['id']) + since = now_iso() + worker_with('before_ack') + wait_log('worker', since, '"point":"before_ack"', 60) + wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 60) + status, attempt, _ = job_row(att['id']) + check(status == 'ready' and attempt == 1 and variant_count(att['id']) == 3, 'redelivery after commit changed result') + check(broker('XPENDING', 'attachments.jobs', 'workers').splitlines()[0] == '0', 'redelivered message not acked') + check(wait_status(c, bob, att['id'], ('ready',), 10)['status'] == 'ready', 'attachment not ready') + restore_worker() + print('3) worker crashed after COMMIT before ack: message redelivered after restart, skipped as no-op, attempt=1, variants=3') + + +def retries_and_replay(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + job_id = job_id_of(att['id']) + compose('stop', 's3') + since = now_iso() + try: + compose('up', '-d', '--no-deps', 'worker') + check('(healthy)' not in compose('ps', '-a', 's3', '--format', '{{.Status}}').stdout, 's3 must stay stopped') + failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 120) + check(failed['status'] == 'failed' and failed['error_code'] == 'retry_exhausted', 'expected retry_exhausted') + text = logs('worker', since) + delays = [int(d) for d in re.findall(r'"job retry scheduled","job_id":"%s".*?"delay_ms":(\d+)' % job_id, text)] + check(len(delays) == 4, 'expected 4 scheduled retries, got %d' % len(delays)) + for n, d in enumerate(delays): + base = 1000 * 2 ** n + check(base // 2 <= d <= base, 'delay %d outside jittered window for retry %d' % (d, n + 1)) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, body={'text': '', 'attachment_ids': [att['id']]}, + headers={'Idempotency-Key': uuid.uuid4().hex}) + check(any(x['attachment_id'] == att['id'] for x in ops('failed')), 'failed registry misses job') + print('4) s3 down: transient errors retried with backoff+jitter delays_ms=%s, then failed retry_exhausted, listed in registry' % delays) + finally: + compose('start', 's3') + wait_s3() + check(ops('replay', att['id'])[0]['result'] == 'queued', 'replay did not queue job') + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 120) + check(ready['status'] == 'ready' and variant_count(att['id']) == 3, 'replayed job did not finish') + check(ops('replay', att['id'])[0]['result'] == 'skipped_ready', 'replay of ready job must be a no-op') + check(psql("select count(*) from job_replays where job_id='%s'" % job_id) == '1', 'replay audit missing') + check(variant_count(att['id']) == 3, 'second replay created new effects') + print(' replay: failed -> queued -> ready (audit rows=1); replay of ready job skipped, variants=3') + + +def permanent_error(c, alice, bob, chat): + data = bytearray(SAMPLE) + mid = len(data) // 2 + data[mid:mid + 64] = b'\x00' * 64 + att = upload(c, alice, chat, bytes(data))['json'] + failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 60) + status, attempt, _ = job_row(att['id']) + check(failed['status'] == 'failed' and failed['error_code'] == 'invalid_image' and attempt == 1, 'corrupt image must fail without retries') + check(failed['variants'] == [], 'failed attachment has variants') + print('5) corrupted IDAT (passes header/tail check in API): failed invalid_image on attempt 1, no retries') + + +def main(): + args = arguments('Fault injection: COMMIT/publish gap, crash before COMMIT and before ack, retries, DLQ replay') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + try: + crash_after_commit(c, alice, bob, chat) + crash_after_store(c, alice, bob, chat) + crash_before_ack(c, alice, bob, chat) + retries_and_replay(c, alice, bob, chat) + permanent_error(c, alice, bob, chat) + finally: + compose('up', '-d', '--no-deps', 's3') + restore_worker() + if 'after_commit' in compose('exec', '-T', 'api', 'printenv', 'FAULT_POINT', check_rc=False).stdout: + restore_api(c) + print('PASS: outbox recovery, lease recovery, redelivery after commit, backoff with jitter, retry_exhausted, replay, permanent error') + + +if __name__ == '__main__': + run(main) diff --git a/tests/idempotency.py b/tests/idempotency.py new file mode 100644 index 0000000..951d438 --- /dev/null +++ b/tests/idempotency.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import threading +import uuid +from client import Client, Failure, arguments, check, run + + +def main(): + args = arguments('Concurrent Idempotency-Key check through the load balancer') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/messages' + parallel = 32 + key = 'idem-' + uuid.uuid4().hex + barrier = threading.Barrier(parallel) + + def send(_): + local = Client(args.base_url, args.ca_file) + barrier.wait() + return local.call('POST', path, 201, actor=alice, body={'text': 'same'}, headers={'Idempotency-Key': key}, schema='Message') + + with ThreadPoolExecutor(max_workers=parallel) as pool: + results = list(pool.map(send, range(parallel))) + ids = {r['json']['id'] for r in results} + instances = {r['headers']['X-Instance-Id'] for r in results} + check(len(ids) == 1, 'parallel requests produced %d messages' % len(ids)) + check(len(instances) >= 2, 'requests were served by one instance only') + check(len({r['json']['created_at'] for r in results}) == 1, 'created_at differs between replays') + + listing = c.call('GET', path, actor=alice, schema='MessagePage')['json']['items'] + check(sum(1 for m in listing if m['text'] == 'same') == 1, 'duplicate message in history') + + c.call('POST', path, 409, actor=alice, body={'text': 'other'}, headers={'Idempotency-Key': key}) + c.call('POST', path, 400, actor=alice, body={'text': 'x'}) + c.call('POST', path, 400, actor=alice, body={'text': 'x'}, headers={'Idempotency-Key': 'bad key!'}) + other = c.call('POST', path, 201, actor=bob, body={'text': 'same'}, headers={'Idempotency-Key': key}, schema='Message')['json'] + check(other['id'] not in ids, 'key must be scoped per user') + print('PASS: %d parallel requests, 1 message, %d instances' % (parallel, len(instances))) + + +if __name__ == '__main__': + run(main) diff --git a/tests/image_gen.py b/tests/image_gen.py new file mode 100644 index 0000000..aeb7de0 --- /dev/null +++ b/tests/image_gen.py @@ -0,0 +1,23 @@ +import os +import struct +import zlib + + +def chunk(kind, data): + body = kind + data + return struct.pack('>I', len(data)) + body + struct.pack('>I', zlib.crc32(body) & 0xffffffff) + + +def png(width, height, noise=False): + header = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + raw = bytearray() + row = width * 3 + 1 + for _ in range(height): + raw += b'\x00' + (os.urandom(row - 1) if noise else bytes(row - 1)) + data = zlib.compress(bytes(raw), 0 if noise else 6) + return header + chunk(b'IDAT', data) + chunk(b'IEND', b'') + + +def png_header_only(width, height): + header = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + return header + chunk(b'IDAT', zlib.compress(b'\x00')) + chunk(b'IEND', b'') diff --git a/tests/incidents.py b/tests/incidents.py new file mode 100644 index 0000000..835c7ff --- /dev/null +++ b/tests/incidents.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +import subprocess +import threading +import time +import uuid +from client import Client, Failure, arguments, check, run +from obs import alerts, loki, prom, prom_value, trace_search, wait_alert, wait_trace_spans +from stack import compose, restore_worker, upload, wait_status + + +class Load: + def __init__(self, c, actor, chat, uploads=False, workers=3): + self.c, self.actor, self.chat, self.uploads = c, actor, chat, uploads + self.stop = threading.Event() + self.codes = {} + self.lock = threading.Lock() + self.threads = [threading.Thread(target=self.loop, args=(i,)) for i in range(workers)] + + def record(self, code): + with self.lock: + self.codes[code] = self.codes.get(code, 0) + 1 + + def loop(self, index): + while not self.stop.is_set(): + try: + if self.uploads and index == 0: + self.c.call('POST', '/api/v1/chats/%s/attachments?filename=l-%s.png' % (self.chat['id'], uuid.uuid4().hex[:8]), + [202, 503], actor=self.actor, raw=open('tests/fixtures/sample.png', 'rb').read(), + headers={'Content-Type': 'image/png'}) + self.record('upload') + time.sleep(1) + continue + r = self.c.call('POST', '/api/v1/chats/%s/messages' % self.chat['id'], [201, 503], actor=self.actor, + body={'text': 'load ' + uuid.uuid4().hex}, headers={'Idempotency-Key': uuid.uuid4().hex}) + self.record(r['status']) + except Failure as exc: + self.record('failure') + time.sleep(0.2) + + def __enter__(self): + for t in self.threads: + t.start() + return self + + def __exit__(self, *exc): + self.stop.set() + for t in self.threads: + t.join() + + +def rss_mib(container): + out = subprocess.run(['docker', 'stats', '--no-stream', '--format', '{{.MemUsage}}', container], capture_output=True, text=True).stdout + value = out.split('/')[0].strip() + number = float(''.join(ch for ch in value if ch.isdigit() or ch == '.')) + return number * (1024 if value.endswith('GiB') else 1 if value.endswith('MiB') else 1 / 1024) + + +def worker_stopped(c, alice, bob, chat): + check(not alerts().get('QueueLagGrowing'), 'alert must start resolved') + compose('stop', 'worker') + stopped = time.time() + with Load(c, alice, chat, uploads=True, workers=1): + fired_after = wait_alert('QueueLagGrowing', True, 180) + age = prom_value('max(queue_oldest_job_age_seconds)') + depth = prom_value('max(queue_depth)') + rate = prom_value('sum(rate(jobs_processed_total[1m]))', 0) + busy = prom_value('sum(worker_busy)') + stuck = trace_search('{ resource.service.name = "messenger-api" && name = "attachments.jobs publish" }', 5, stopped)[0] + spans = wait_trace_spans(stuck['traceID'], lambda s: any(x['name'] == 'attachments.jobs publish' for x in s)) + consumers = [s for s in spans if s['kind'] == 'SPAN_KIND_CONSUMER'] + silent = loki('{service_name="messenger-worker"} | event="job ready"', 1, 5) + check(not consumers, 'stuck job already has a consumer span') + check(age is not None and age > 20 and depth and depth > 0, 'queue lag not visible in metrics') + check(rate < 0.01, 'jobs still processed while worker is stopped') + restart = time.monotonic() + restore_worker() + resolved_after = wait_alert('QueueLagGrowing', False, 180) + deadline = time.monotonic() + 120 + while prom_value('max(queue_depth)', 0) > 0 and time.monotonic() < deadline: + time.sleep(2) + drained = time.monotonic() - restart + check(prom_value('max(queue_depth)', 0) == 0, 'backlog did not drain') + print('1) worker stopped: QueueLagGrowing firing after %.0fs; oldest job age=%.0fs depth=%d jobs/s=%.2f worker_busy=%s' + % (fired_after, age, depth, rate, busy)) + print(' diagnosis from stored data: trace %s has the api server and "attachments.jobs publish" spans and no consumer span; ' + 'no "job ready" worker log in Loki during the last minute (%d lines)' % (stuck['traceID'], len(silent))) + print(' restored: worker started, alert resolved after %.0fs, backlog drained in %.0fs' % (resolved_after, drained)) + + +def slow_database(c, alice, bob, chat): + check(not alerts().get('ApiHighErrorRate'), 'alert must start resolved') + lock = subprocess.Popen(['docker', 'compose', '--env-file', '.env', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', + '-c', 'begin; lock table messages in access exclusive mode; select pg_sleep(45); commit;'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + time.sleep(2) + with Load(c, alice, chat, workers=4) as load: + fired_after = wait_alert('ApiHighErrorRate', True, 120) + ratio = prom_value('sum(rate(http_server_requests_total{http_response_status_class="5xx"}[1m])) / sum(rate(http_server_requests_total[1m]))') + by_route = {r['metric']['http_route']: float(r['value'][1]) for r in + prom('sum by (http_route) (rate(http_server_requests_total{http_response_status_class="5xx"}[1m]))') if float(r['value'][1]) > 0} + p99 = prom_value('histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket[1m])))') + used = prom_value('sum(db_pool_connections{state="used",job="messenger-api"})') + wait = prom_value('sum(rate(db_pool_acquire_wait_seconds_total{job="messenger-api"}[1m]))', 0) + errors = trace_search('{ resource.service.name = "messenger-api" && status = error && name =~ "POST .*messages" }', 5) + check(errors, 'no error trace found') + spans = wait_trace_spans(errors[0]['traceID'], lambda s: len(s) > 1) + server = [s for s in spans if s['kind'] == 'SPAN_KIND_SERVER'][0] + slow_db = max((s for s in spans if s['name'].startswith('db ')), key=lambda s: s['duration_us']) + share = slow_db['duration_us'] / server['duration_us'] + check(ratio and ratio > 0.05, 'error ratio not visible') + check(share > 0.8 and slow_db['status'] == 'STATUS_CODE_ERROR', 'time is not spent in the database span') + check(set(by_route) <= {'/api/v1/chats/{chat_id}/messages'}, 'unexpected failing routes: %s' % by_route) + lock.wait(timeout=90) + resolved_after = wait_alert('ApiHighErrorRate', False, 180) + print('2) database lock on messages: ApiHighErrorRate firing after %.0fs; 5xx ratio=%.2f only on %s; p99=%.1fs; pool used=%s acquire wait=%.2f s/s' + % (fired_after, ratio, sorted(by_route), p99 or 0, used, wait)) + print(' diagnosis from stored data: error trace %s, server span %d ms, child "%s" %d ms (%.0f%%) with status error, ' + 'handler has no other slow child; requests: %s' % (errors[0]['traceID'], server['duration_us'] // 1000, slow_db['name'], + slow_db['duration_us'] // 1000, share * 100, load.codes)) + print(' restored: lock released, alert resolved after %.0fs' % resolved_after) + + +def storage_down(c, alice, bob, chat): + stores = ['prometheus', 'loki', 'tempo'] + container = compose('ps', '-q', 'otel-collector').stdout.strip() + before = {name: prom_value('sum(%s)' % name, 0) for name in + ('otelcol_exporter_send_failed_spans', 'otelcol_exporter_send_failed_log_records', 'otelcol_exporter_send_failed_metric_points')} + peak = rss_mib(container) + compose('stop', *stores) + samples = [] + attachments = [] + try: + with Load(c, alice, chat, uploads=True, workers=3) as load: + for _ in range(20): + time.sleep(4) + samples.append(rss_mib(container)) + codes = dict(load.codes) + for _ in range(3): + attachments.append(upload(c, alice, chat)['json']['id']) + ready = [wait_status(c, bob, a, ('ready', 'failed'), 60)['status'] for a in attachments] + msg = c.message(alice, chat, 'during outage') + state = subprocess.run(['docker', 'inspect', '-f', '{{.RestartCount}} {{.State.OOMKilled}}', container], capture_output=True, text=True).stdout.split() + finally: + compose('up', '-d', '--wait', *stores) + check(codes.get(201, 0) > 20 and 'failure' not in codes and 503 not in codes, 'API was affected by telemetry outage: %s' % codes) + check(ready == ['ready'] * 3 and msg['status'] == 201, 'messages or jobs affected by telemetry outage') + check(state == ['0', 'false'], 'collector restarted or was OOM killed') + check(max(samples) < 230, 'collector memory unbounded: %.0f MiB' % max(samples)) + time.sleep(30) + after = {name: prom_value('sum(%s)' % name, 0) for name in before} + check(after['otelcol_exporter_send_failed_spans'] > before['otelcol_exporter_send_failed_spans'], 'collector did not count dropped spans') + exports = prom_value('sum(telemetry_export_errors_total)', 0) + fresh = loki('{service_name="messenger-api"} | event="request"', 2, 5) + check(fresh, 'logs did not resume after storage recovery') + print('3) prometheus, loki and tempo stopped for ~80s under load: requests=%s, 3/3 uploads ready, message 201, no 5xx' % codes) + print(' collector rss MiB: start=%.0f peak=%.0f during outage (limit 256, memory_limiter 160), restarts/oom=%s' % (peak, max(samples), ' '.join(state))) + print(' telemetry lost and counted: collector send_failed spans +%d, log records +%d, metric points +%d; sdk export errors=%d' + % (after['otelcol_exporter_send_failed_spans'] - before['otelcol_exporter_send_failed_spans'], + after['otelcol_exporter_send_failed_log_records'] - before['otelcol_exporter_send_failed_log_records'], + after['otelcol_exporter_send_failed_metric_points'] - before['otelcol_exporter_send_failed_metric_points'], exports)) + print(' after restore new logs and metrics are stored again; data from the outage window is lost, data before it remains') + + +def main(): + def configure(p): + p.add_argument('--only', choices=['worker', 'database', 'storage']) + args = arguments('Controlled incidents with alerts and diagnosis from stored telemetry', configure) + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + try: + if args.only in (None, 'worker'): + worker_stopped(c, alice, bob, chat) + if args.only in (None, 'database'): + slow_database(c, alice, bob, chat) + if args.only in (None, 'storage'): + storage_down(c, alice, bob, chat) + finally: + restore_worker() + compose('up', '-d', '--no-deps', 'prometheus', 'loki', 'tempo', check_rc=False) + print('PASS: ' + {'worker': 'stopped worker detected by alert and diagnosed from telemetry', + 'database': 'slow database detected by alert and diagnosed from telemetry', + 'storage': 'telemetry storage outage does not affect the API and loss is bounded and counted', + None: 'two alerting incidents diagnosed from telemetry, storage outage does not affect the API'}[args.only]) + + +if __name__ == '__main__': + run(main) diff --git a/tests/obs.py b/tests/obs.py new file mode 100644 index 0000000..daf6ae6 --- /dev/null +++ b/tests/obs.py @@ -0,0 +1,125 @@ +import json +import time +import urllib.parse +import urllib.request +from client import Failure, check + +PROM = 'http://127.0.0.1:9090' +LOKI = 'http://127.0.0.1:3100' +TEMPO = 'http://127.0.0.1:3200' + + +def get(url, params=None, timeout=10): + if params: + url += '?' + urllib.parse.urlencode(params) + with urllib.request.urlopen(url, timeout=timeout) as response: + return json.loads(response.read()) + + +def prom(expr): + data = get(PROM + '/api/v1/query', {'query': expr}) + check(data['status'] == 'success', 'prometheus query failed: ' + expr) + return data['data']['result'] + + +def prom_value(expr, default=None): + result = prom(expr) + if not result: + return default + value = float(result[0]['value'][1]) + return default if value != value else value + + +def series(name): + return get(PROM + '/api/v1/series', {'match[]': name})['data'] + + +def alerts(): + return {a['labels']['alertname']: a['state'] for a in get(PROM + '/api/v1/alerts')['data']['alerts']} + + +def wait_alert(name, firing, timeout=120): + deadline = time.monotonic() + timeout + started = time.monotonic() + while time.monotonic() < deadline: + state = alerts().get(name) + if (state == 'firing') == firing: + return time.monotonic() - started + time.sleep(2) + raise Failure('alert %s did not become %s' % (name, 'firing' if firing else 'resolved')) + + +def loki(query, minutes=15, limit=200): + end = time.time() + data = get(LOKI + '/loki/api/v1/query_range', { + 'query': query, 'limit': limit, 'direction': 'backward', + 'start': int((end - minutes * 60) * 1e9), 'end': int((end + 60) * 1e9)}) + rows = [] + for stream in data['data']['result']: + for ts, line in stream['values']: + rows.append((int(ts), stream['stream'], line)) + return sorted(rows, key=lambda r: r[0]) + + +def wait_loki(query, timeout=40, **kw): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + rows = loki(query, **kw) + if rows: + return rows + time.sleep(2) + raise Failure('loki returned nothing for ' + query) + + +def trace(trace_id, timeout=40): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + return flatten(get(TEMPO + '/api/traces/' + trace_id)) + except Exception: + time.sleep(2) + raise Failure('tempo has no trace ' + trace_id) + + +def attrs(items): + out = {} + for a in items: + v = a['value'] + out[a['key']] = v.get('stringValue', v.get('intValue', v.get('boolValue', v.get('doubleValue')))) + return out + + +def flatten(data): + spans = [] + for batch in data.get('batches', []): + service = attrs(batch['resource']['attributes']).get('service.name') + for scope in batch['scopeSpans']: + for s in scope['spans']: + spans.append({ + 'service': service, 'name': s['name'], 'kind': s['kind'], 'id': s['spanId'], + 'parent': s.get('parentSpanId', ''), 'start': int(s['startTimeUnixNano']), + 'duration_us': (int(s['endTimeUnixNano']) - int(s['startTimeUnixNano'])) // 1000, + 'attributes': attrs(s.get('attributes', [])), 'status': s.get('status', {}).get('code', ''), + }) + return sorted(spans, key=lambda s: s['start']) + + +def trace_search(query, limit=20, since=None): + params = {'q': query, 'limit': limit} + if since: + params.update(start=int(since), end=int(time.time()) + 60) + return get(TEMPO + '/api/search', params).get('traces', []) + + +def wait_trace_spans(trace_id, predicate, timeout=60): + deadline = time.monotonic() + timeout + spans = [] + while time.monotonic() < deadline: + try: + spans = trace(trace_id, 5) + except Failure: + spans = [] + if spans and predicate(spans): + return spans + time.sleep(2) + raise Failure('trace %s did not reach the expected shape' % trace_id) diff --git a/tests/observability.py b/tests/observability.py new file mode 100644 index 0000000..2cacad0 --- /dev/null +++ b/tests/observability.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +import json +import re +import time +import uuid +from client import Client, arguments, check, run +from faults import ops, wait_s3 +from obs import loki, prom, series, wait_loki, wait_trace_spans +from stack import compose, restore_worker, upload, wait_status + +SIGNALS = [ + 'http_server_requests_total', 'http_server_request_duration_seconds_bucket', 'jobs_processed_total', + 'job_processing_duration_seconds_bucket', 'job_time_to_ready_seconds_bucket', 'queue_depth', + 'queue_oldest_job_age_seconds', 'db_pool_connections', 'db_pool_max_connections', 'worker_busy', + 'worker_capacity', 'process_cpu_time_seconds_total', 'process_memory_usage_bytes', +] +FORBIDDEN = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'filename', 'url', 'http_url', 'path', 'text'} +UUID = re.compile(r'[0-9a-f]{8}-[0-9a-f]{4}-') + + +def signals_and_labels(): + for name in SIGNALS: + check(prom('count(%s)' % name), 'metric %s has no samples' % name) + seen = set() + for name in SIGNALS: + for s in series(name): + seen |= set(s) - {'__name__'} + check(not UUID.search(s.get('http_route', '')), 'raw identifier in route label') + check(not (seen & FORBIDDEN), 'high-cardinality labels present: %s' % sorted(seen & FORBIDDEN)) + routes = {s['http_route'] for s in series('http_server_requests_total')} + check('/api/v1/chats/{chat_id}/messages' in routes, 'route template missing') + print('metrics: %d signals present, labels=%s, routes are templates (%d)' % (len(SIGNALS), sorted(seen), len(routes))) + + +def chain(c, alice, bob, chat): + resp = upload(c, alice, chat) + request_id = resp['headers']['X-Request-Id'] + att = resp['json'] + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + check(ready['status'] == 'ready', 'attachment did not become ready') + rows = wait_loki('{service_name="messenger-api"} | event="request" | request_id="%s"' % request_id) + trace_id = rows[-1][1].get('trace_id') + check(trace_id and re.fullmatch(r'[0-9a-f]{32}', trace_id), 'request log has no trace_id') + check(rows[-1][1].get('span_id'), 'request log has no span_id') + spans = wait_trace_spans(trace_id, lambda s: any(x['kind'] == 'SPAN_KIND_CONSUMER' for x in s) + and any(x['name'] == 'db COMMIT' and x['service'] == 'messenger-worker' for x in s) + and any(x['name'] == 's3 remove' for x in s)) + by_id = {s['id']: s for s in spans} + server = [s for s in spans if s['kind'] == 'SPAN_KIND_SERVER'][0] + producer = [s for s in spans if s['kind'] == 'SPAN_KIND_PRODUCER'][0] + consumer = [s for s in spans if s['kind'] == 'SPAN_KIND_CONSUMER'][0] + check(server['service'] == 'messenger-api' and producer['service'] == 'messenger-api', 'producer must belong to api') + check(consumer['service'] == 'messenger-worker', 'consumer must belong to worker') + check(producer['parent'] == server['id'], 'producer span is not a child of the request span') + check(consumer['parent'] == producer['id'], 'consumer span is not a child of the producer span') + check(consumer['attributes'].get('attachment.id') == att['id'], 'consumer span misses attachment.id') + names = {(s['service'], s['name']) for s in spans} + for want in [('messenger-api', 's3 put'), ('messenger-api', 'db INSERT'), ('messenger-worker', 's3 get'), + ('messenger-worker', 'image render'), ('messenger-worker', 's3 put'), ('messenger-worker', 'db COMMIT'), + ('messenger-worker', 's3 remove')]: + check(want in names, 'span %s missing' % (want,)) + check(all(s['parent'] in by_id or not s['parent'] for s in spans), 'orphan spans in trace') + worker_logs = wait_loki('{service_name="messenger-worker"} | event="job ready" | trace_id="%s"' % trace_id) + check(worker_logs[-1][1].get('span_id'), 'worker log has no span_id') + print('chain: request_id=%s trace_id=%s spans=%d api->producer->consumer->s3/db linked, api and worker logs carry trace_id' + % (request_id, trace_id, len(spans))) + return att, trace_id + + +def secrets(c, alice, chat, trace_id): + marker = 'secret-body-' + uuid.uuid4().hex + resp = c.message(alice, chat, marker) + request_id = resp['headers']['X-Request-Id'] + time.sleep(8) + rows = wait_loki('{service_name="messenger-api"} | event="request" | request_id="%s"' % request_id) + message_trace = rows[-1][1].get('trace_id') + raw = json.dumps(wait_trace_spans(message_trace, lambda s: len(s) > 1)) + json.dumps(wait_trace_spans(trace_id, lambda s: True)) + for needle, what in ((marker, 'message body'), (alice['token'], 'session token')): + check(needle not in raw, '%s leaked into traces' % what) + check(not loki('{service_name=~"messenger-.*"} |= "%s"' % needle), '%s leaked into logs' % what) + check(not loki('{service_name=~"messenger-.*"} |~ "(?i)authorization|bearer "'), 'authorization material in logs') + print('secrets: message body and session token absent from Loki and Tempo') + + +def retry_trace(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + compose('stop', 's3') + try: + compose('up', '-d', '--no-deps', 'worker') + failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 120) + check(failed['status'] == 'failed', 'expected retry_exhausted while S3 is down') + finally: + compose('start', 's3') + wait_s3() + check(ops('replay', att['id'])[0]['result'] == 'queued', 'replay did not queue job') + check(wait_status(c, bob, att['id'], ('ready', 'failed'), 120)['status'] == 'ready', 'replay did not finish') + rows = wait_loki('{service_name="messenger-worker"} | event="job ready" | attachment_id="%s"' % att['id']) + trace_id = rows[-1][1]['trace_id'] + spans = wait_trace_spans(trace_id, lambda s: sum(x['kind'] == 'SPAN_KIND_CONSUMER' for x in s) >= 6 + and any(x['name'] == 'db COMMIT' for x in s)) + consumers = [s for s in spans if s['kind'] == 'SPAN_KIND_CONSUMER'] + attempts = sorted(int(s['attributes']['job.attempt']) for s in consumers) + check(attempts == list(range(1, len(attempts) + 1)), 'attempts are not sequential: %s' % attempts) + errors = [s for s in consumers if s['status'] == 'STATUS_CODE_ERROR'] + check(len(errors) == 5, 'expected 5 failed attempts in the trace, got %d' % len(errors)) + check(sum(1 for s in spans if s['kind'] == 'SPAN_KIND_PRODUCER') == len(consumers), 'each attempt needs its publish span') + print('retry: one trace %s holds %d attempts (%d failed with S3 down, attempt %d ready after replay)' + % (trace_id, len(consumers), len(errors), attempts[-1])) + + +def main(): + args = arguments('Observability signals, trace propagation, correlation and redaction') + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + try: + att, trace_id = chain(c, alice, bob, chat) + signals_and_labels() + secrets(c, alice, chat, trace_id) + retry_trace(c, alice, bob, chat) + finally: + compose('start', 's3', check_rc=False) + restore_worker() + print('PASS: signals, producer/consumer trace chain, log correlation, retry in trace, no secrets in telemetry') + + +if __name__ == '__main__': + run(main) diff --git a/tests/queue_recovery.py b/tests/queue_recovery.py new file mode 100755 index 0000000..0c420d8 --- /dev/null +++ b/tests/queue_recovery.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +import json +import time +from client import Client, arguments, check, run +from image_probe import jpeg_size +from stack import broker, compose, job_row, logs, now_iso, psql, upload, variant_count, wait_api, wait_log, wait_status + + +def stream_has(job_id): + return job_id in broker('XRANGE', 'attachments.jobs', '-', '+') + + +def main(): + args = arguments('Worker stop, API/broker restart and redelivery scenario') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + + compose('stop', 'worker') + try: + att = upload(c, alice, chat)['json'] + check(att['status'] == 'queued' and att['variants'] == [], 'upload must return queued snapshot') + msg = c.message(alice, chat, '', attachments=[att['id']])['json'] + check(msg['attachment_ids'] == [att['id']], 'message lost queued attachment') + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 409, actor=bob) + c.message(bob, chat, 'text works while worker is down') + job_id = psql("select id from jobs where attachment_id='%s'" % att['id']) + deadline = time.monotonic() + 10 + while not stream_has(job_id): + check(time.monotonic() < deadline, 'job was not published to broker') + time.sleep(0.5) + check(psql("select count(*) from outbox where job_id='%s' and published_at is null" % job_id) == '0', 'outbox not marked') + print('accepted: attachment=%s job=%s status=queued, job in stream, worker stopped' % (att['id'], job_id)) + + compose('restart', 'broker', 'api') + compose('up', '-d', '--wait', '--no-deps', 'broker', 'api') + compose('restart', 'lb') + wait_api(c) + check(stream_has(job_id), 'job lost after broker restart') + check(c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status'] == 'queued', 'status changed without worker') + print('restarted broker and api: job still in stream, attachment still queued') + finally: + compose('start', 'worker') + + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + check(ready['status'] == 'ready', 'job did not finish after worker start') + thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob) + check(jpeg_size(thumb['bytes']) == (256, 160), 'thumbnail size') + check(job_row(att['id'])[:2] == ('ready', 1) and variant_count(att['id']) == 3, 'unexpected job state or variants') + print('worker started: attachment ready, attempt=1, variants=3') + + since = now_iso() + before = psql("select updated_at from attachments where id='%s'" % att['id']) + payload = json.dumps({'schema_version': 1, 'job_id': job_id, 'type': 'attachment.process.v1', 'attachment_id': att['id'], + 'pipeline_version': 1, 'attempt': 1, 'created_at': ready['created_at']}) + broker('XADD', 'attachments.jobs', '*', 'job', payload) + wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 30) + time.sleep(1) + check(psql("select updated_at from attachments where id='%s'" % att['id']) == before, 'duplicate delivery changed attachment') + check(variant_count(att['id']) == 3 and job_row(att['id'])[:2] == ('ready', 1), 'duplicate delivery created new effects') + pending = broker('XPENDING', 'attachments.jobs', 'workers') + check(pending.splitlines()[0] == '0', 'duplicate message was not acked') + print('duplicate delivery: skipped as no-op, acked, variants=3, updated_at unchanged') + print('PASS: worker stop, broker/api restart, redelivery idempotency') + + +if __name__ == '__main__': + run(main) diff --git a/tests/s3_checks.py b/tests/s3_checks.py new file mode 100644 index 0000000..2371df2 --- /dev/null +++ b/tests/s3_checks.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +import hashlib +import subprocess +import time +import uuid +from pathlib import Path +from client import Client, Failure, arguments, check, run +from stack import wait_status + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() +BUCKET = 'messenger-attachments' + + +def compose(*args, check_rc=True, stdin=None): + return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=check_rc, + capture_output=True, text=True, input=stdin) + + +def secret(name): + return (ROOT / 'secrets' / name).read_text().strip() + + +def s3_status(method, path, user=None, data=None): + cmd = ['exec', '-T', 's3', 'curl', '-s', '-o', '/dev/null', '-w', '%{http_code}', '-X', method] + if user: + cmd += ['--aws-sigv4', 'aws:amz:us-east-1:s3', '--user', user[0] + ':' + user[1]] + if data is not None: + cmd += ['--data-binary', data] + cmd.append('http://127.0.0.1:9000' + path) + return compose(*cmd).stdout.strip() + + +def upload(c, actor, chat, expected): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=a.png', expected, actor=actor, + raw=SAMPLE, headers={'Content-Type': 'image/png'}, schema='Attachment' if expected == 202 else None) + + +def wait_api(c): + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + try: + c.call('GET', '/health/ready') + return + except Failure: + time.sleep(1) + raise Failure('api did not recover') + + +def wait_s3(timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if compose('ps', 's3', '--format', '{{.Status}}').stdout.count('(healthy)'): + return + time.sleep(1) + raise Failure('s3 did not recover') + + +def object_key(attachment_id): + out = compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', + "select object_key from attachments where id='%s'" % attachment_id).stdout.strip() + check(out.startswith('attachments/'), 'object key not found') + return out + + +def main(): + args = arguments('S3 privacy, outage and orphan cleanup checks') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + att = upload(c, alice, chat, 202)['json'] + wait_status(c, bob, att['id'], ('ready',), 60) + key = object_key(att['id']) + check(key != 'attachments/' + att['filename'] and att['filename'] not in key, 'object key derived from filename') + + api = (secret('s3_access_key_id'), secret('s3_secret_access_key')) + admin = (secret('s3_admin_access_key_id'), secret('s3_admin_secret_access_key')) + path = '/' + BUCKET + '/' + key + check(s3_status('GET', path) == '403', 'anonymous GET must be denied') + check(s3_status('GET', '/' + BUCKET + '/') == '403', 'anonymous listing must be denied') + check(s3_status('PUT', path + '-anon', data='x') == '403', 'anonymous PUT must be denied') + check(s3_status('GET', path, ('wrong', 'wrong')) == '403', 'wrong credentials must be denied') + check(s3_status('GET', path, api) == '200', 'api credentials cannot read') + check(s3_status('PUT', '/other-bucket-' + uuid.uuid4().hex[:8], api) == '403', 'api credentials can create buckets') + ports = compose('ps', '--format', '{{.Service}} {{.Ports}}').stdout + lines = [line for line in ports.splitlines() if '->' in line] + published = [line.split()[0] for line in lines] + check(sorted(published) == ['grafana', 'lb', 'loki', 'prometheus', 'tempo'], 'unexpected published services: %s' % published) + check(all(part.startswith('127.0.0.1:') for line in lines for part in line.split(None, 1)[1].split(', ') if '->' in part), + 'published ports must be bound to 127.0.0.1') + + compose('stop', 's3') + try: + upload(c, alice, chat, 503) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob) + c.call('GET', '/api/v1/attachments/' + att['id'], 200, actor=bob, schema='Attachment') + c.message(alice, chat, 'text works while s3 is down') + finally: + compose('start', 's3') + wait_s3() + wait_api(c) + deadline = time.monotonic() + 60 + while True: + try: + body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] + check(body == SAMPLE, 'content differs after s3 restart') + break + except Failure: + check(time.monotonic() < deadline, 'download did not recover') + time.sleep(1) + upload(c, alice, chat, 202) + + orphan = '/' + BUCKET + '/attachments/' + uuid.uuid4().hex + check(s3_status('PUT', orphan, admin, 'orphan') == '200', 'cannot create orphan') + compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=3600', 'janitor', 'cleanup') + check(s3_status('GET', orphan, admin) == '200', 'fresh object was removed inside grace period') + time.sleep(1) + compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=0', 'janitor', 'cleanup') + check(s3_status('GET', orphan, admin) == '404', 'orphan was not removed') + check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == SAMPLE, 'referenced object was removed') + + check(s3_status('GET', '/' + BUCKET + '/staging/' + att['id'], admin) == '404', 'staging kept after ready') + stale = '/' + BUCKET + '/staging/' + str(uuid.uuid4()) + check(s3_status('PUT', stale, admin, 'stale') == '200', 'cannot create stale staging object') + compose('stop', 'worker') + try: + queued = upload(c, alice, chat, 202)['json'] + compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=0', 'janitor', 'cleanup') + check(s3_status('GET', '/' + BUCKET + '/staging/' + queued['id'], admin) == '200', 'staging of queued job was removed') + check(s3_status('GET', stale, admin) == '404', 'unreferenced staging object was not removed') + finally: + compose('up', '-d', '--no-deps', 'worker') + check(wait_status(c, bob, queued['id'], ('ready', 'failed'), 60)['status'] == 'ready', 'queued job lost staging') + print('PASS: private bucket, limited credentials, s3 outage, orphan and staging cleanup') + + +if __name__ == '__main__': + run(main) diff --git a/tests/scenario.py b/tests/scenario.py new file mode 100644 index 0000000..f7ddd70 --- /dev/null +++ b/tests/scenario.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import json +import subprocess +import threading +import time +import uuid +from pathlib import Path +from client import Client, Failure, arguments, check, run + +ROOT = Path(__file__).resolve().parent.parent + + +def sh(*cmd): + return subprocess.run(cmd, cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip() + + +def api_containers(): + out = sh('docker', 'compose', '--env-file', '.env', 'ps', '-q', 'api') + return [x[:12] for x in out.split()] + + +def wait_healthy(count, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + out = sh('docker', 'compose', '--env-file', '.env', 'ps', 'api', '--format', '{{.Status}}') + if out.count('(healthy)') >= count: + return + time.sleep(1) + raise Failure('replicas did not become healthy') + + +class Load: + def __init__(self, base, ca, concurrency): + c = Client(base, ca) + self.alice, _, _, self.chat = c.bootstrap() + self.base, self.ca, self.n = base, ca, concurrency + self.events = [] + self.lock = threading.Lock() + self.stop = threading.Event() + self.t0 = time.monotonic() + + def worker(self): + c = Client(self.base, self.ca, timeout=3) + while not self.stop.is_set(): + key = uuid.uuid4().hex + first_error = None + while not self.stop.is_set(): + try: + r = c.message(self.alice, self.chat, 'k-' + key, key=key) + t = time.monotonic() - self.t0 + with self.lock: + self.events.append((t, True, r['headers']['X-Instance-Id'], first_error)) + break + except Failure: + t = time.monotonic() - self.t0 + first_error = first_error or t + with self.lock: + self.events.append((t, False, None, None)) + time.sleep(0.05) + + def start(self): + self.pool = ThreadPoolExecutor(max_workers=self.n) + self.futures = [self.pool.submit(self.worker) for _ in range(self.n)] + + def finish(self): + self.stop.set() + self.pool.shutdown(wait=True) + + def summary(self, begin, end): + ev = [e for e in self.events if begin <= e[0] < end] + ok = [e for e in ev if e[1]] + bad = [e for e in ev if not e[1]] + span = max(end - begin, 1e-9) + return { + 'window_seconds': round(span, 2), + 'successful_requests': len(ok), + 'failed_attempts': len(bad), + 'successful_rps': round(len(ok) / span, 1), + 'last_error_offset_seconds': round(max(e[0] for e in bad) - begin, 2) if bad else None, + 'instances': sorted({e[2] for e in ok}), + } + + def instances_between(self, begin, end=float('inf')): + return sorted({e[2] for e in self.events if e[1] and begin <= e[0] < end}) + + +def failover_each(args): + ids = api_containers() + check(len(ids) >= 2, 'need at least two replicas') + load = Load(args.base_url, args.ca_file, args.concurrency) + load.start() + time.sleep(args.warmup) + results = [] + base = load.summary(2, args.warmup) + for target in ids: + before = time.monotonic() - load.t0 + sh('docker', 'stop', '-t', '10', target) + stopped = time.monotonic() - load.t0 + time.sleep(args.hold) + after_stop = time.monotonic() - load.t0 + window = load.summary(before, after_stop) + sh('docker', 'start', target) + wait_healthy(len(ids)) + restored = time.monotonic() - load.t0 + time.sleep(args.warmup) + results.append({ + 'stopped_replica': target, + 'during_failure': window, + 'seen_stopped_replica_after_stop': target in load.instances_between(stopped + 0.5, after_stop), + 'stop_command_seconds': round(stopped - before, 2), + 'restore_seconds': round(restored - after_stop, 2), + 'after_restore': load.summary(restored + 1, restored + args.warmup), + }) + load.finish() + return {'baseline': base, 'replicas': results} + + +def rolling(args): + old = api_containers() + check(len(old) >= 2, 'need at least two replicas') + n = len(old) + load = Load(args.base_url, args.ca_file, args.concurrency) + load.start() + time.sleep(args.warmup) + begin = time.monotonic() - load.t0 + sh('scripts/rolling-update.sh') + end = time.monotonic() - load.t0 + time.sleep(args.warmup) + load.finish() + return { + 'old_replicas': old, + 'new_replicas': api_containers(), + 'update_window': load.summary(begin, end), + 'old_replicas_served_after_update': [i for i in load.instances_between(end) if i in old], + } + + +def main(): + def configure(p): + p.add_argument('--mode', choices=['failover-each', 'rolling'], required=True) + p.add_argument('--concurrency', type=int, default=8) + p.add_argument('--warmup', type=float, default=8) + p.add_argument('--hold', type=float, default=12) + p.add_argument('--output') + args = arguments('Failure and rolling update under load', configure) + report = failover_each(args) if args.mode == 'failover-each' else rolling(args) + text = json.dumps(report, indent=2) + if args.output: + Path(args.output).write_text(text + '\n') + print(text) + + +if __name__ == '__main__': + run(main) diff --git a/tests/session_revocation.py b/tests/session_revocation.py new file mode 100644 index 0000000..2e2a80d --- /dev/null +++ b/tests/session_revocation.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import time +from client import Client, arguments, check, run +from stack import compose + + +def wait_valkey(timeout=60): + deadline = time.monotonic() + timeout + while '(healthy)' not in compose('ps', 'valkey', '--format', '{{.Status}}').stdout: + check(time.monotonic() < deadline, 'valkey did not recover') + time.sleep(1) + + +def main(): + args = arguments('Session revocation must not depend on the cache') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + c.call('GET', '/api/v1/chats', 200, actor=alice) + compose('stop', 'valkey') + try: + c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice) + finally: + compose('start', 'valkey') + wait_valkey() + c.call('GET', '/api/v1/chats', 401, actor=alice) + c.call('GET', '/api/v1/chats', 200, actor=bob) + print('PASS: token revoked while valkey was stopped stays revoked after valkey restart (401)') + + +if __name__ == '__main__': + run(main) diff --git a/tests/stack.py b/tests/stack.py new file mode 100644 index 0000000..f0b645f --- /dev/null +++ b/tests/stack.py @@ -0,0 +1,92 @@ +import os +import subprocess +import time +import uuid +from pathlib import Path +from client import Failure, check + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() + + +def compose(*args, env=None, check_rc=True): + full = dict(os.environ, **(env or {})) + return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=check_rc, + capture_output=True, text=True, env=full) + + +def psql(sql): + return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', sql).stdout.strip() + + +def broker(*args): + cmd = 'valkey-cli --no-auth-warning -a "$(sed -n "s/^requirepass //p" /run/secrets/broker_conf)" ' + ' '.join("'%s'" % a for a in args) + return compose('exec', '-T', 'broker', 'sh', '-c', cmd).stdout.strip() + + +def logs(service, since): + return compose('logs', '--no-color', '--since', since, service).stdout + + +def now_iso(): + return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) + + +def upload(c, actor, chat, data=SAMPLE, content_type='image/png', filename=None, expected=202): + filename = filename or 'f-' + uuid.uuid4().hex[:12] + '.png' + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, expected, actor=actor, + raw=data, headers={'Content-Type': content_type}, schema='Attachment' if expected == 202 else None) + + +def wait_status(c, actor, att_id, statuses, timeout=120): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + att = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json'] + if att['status'] in statuses: + return att + except Failure: + pass + time.sleep(0.5) + raise Failure('attachment %s did not reach %s' % (att_id, statuses)) + + +def wait_api(c, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + c.call('GET', '/health/ready') + return + except Failure: + time.sleep(1) + raise Failure('api did not become ready') + + +def wait_log(service, since, needle, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if needle in logs(service, since): + return + time.sleep(1) + raise Failure('%s did not log %r' % (service, needle)) + + +def job_row(att_id): + out = psql("select status, attempt, coalesce(last_error, '') from jobs where attachment_id='%s'" % att_id) + check(out, 'job not found') + status, attempt, error = out.split('|', 2) + return status, int(attempt), error + + +def variant_count(att_id): + return int(psql("select count(*) from attachment_variants where attachment_id='%s'" % att_id)) + + +def restore_worker(): + compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'WORKER_FAULT_POINT': ''}) + + +def restore_api(c): + compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': ''}) + compose('restart', 'lb') + wait_api(c) diff --git a/tests/telemetry_persistence.py b/tests/telemetry_persistence.py new file mode 100644 index 0000000..3d92225 --- /dev/null +++ b/tests/telemetry_persistence.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +import re +import time +from client import Client, arguments, check, run +from obs import loki, prom, trace, wait_loki +from stack import compose, upload, wait_status + + +def main(): + args = arguments('Telemetry history survives a restart of the storages') + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + request_id = upload(c, alice, chat)['headers']['X-Request-Id'] + rows = wait_loki('{service_name="messenger-api"} | event="request" | request_id="%s"' % request_id) + trace_id = rows[-1][1]['trace_id'] + check(re.fullmatch(r'[0-9a-f]{32}', trace_id), 'no trace_id in log') + time.sleep(15) + before = len(trace(trace_id)) + samples = prom('count_over_time(http_server_requests_total[10m])') + check(before > 1 and samples, 'nothing to compare') + compose('restart', 'prometheus', 'loki', 'tempo') + compose('up', '-d', '--wait', 'prometheus', 'loki', 'tempo', 'grafana') + check(len(trace(trace_id, 90)) == before, 'trace lost after restart') + check(wait_loki('{service_name="messenger-api"} | event="request" | request_id="%s"' % request_id, 90), 'log lost after restart') + check(prom('count_over_time(http_server_requests_total[10m])'), 'metrics lost after restart') + print('persistence: trace %s (%d spans), its request log and %d metric series survived a restart of prometheus, loki and tempo' + % (trace_id, before, len(samples))) + print('PASS: telemetry history is kept on volumes') + + +if __name__ == '__main__': + run(main) diff --git a/tests/upload_memory.py b/tests/upload_memory.py new file mode 100644 index 0000000..aa88929 --- /dev/null +++ b/tests/upload_memory.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import re +import subprocess +import threading +import time +from pathlib import Path +from client import Client, Failure, arguments, check, run +from image_gen import png + +ROOT = Path(__file__).resolve().parent.parent +UNITS = {'B': 1, 'KiB': 1024, 'MiB': 1024 ** 2, 'GiB': 1024 ** 3} + + +def to_mib(text): + value, unit = re.match(r'([\d.]+)\s*([A-Za-z]+)', text).groups() + return float(value) * UNITS[unit] / 1024 ** 2 + + +def sample(stop, peaks): + proc = subprocess.Popen(['docker', 'stats', '--format', '{{.Name}} {{.MemUsage}}'], cwd=ROOT, + stdout=subprocess.PIPE, text=True) + try: + for line in proc.stdout: + line = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line).strip() + m = re.match(r'(\S+-api-\d+)\s+(\S+ ?\S+) /', line) + if m: + peaks[m.group(1)] = max(peaks.get(m.group(1), 0), to_mib(m.group(2))) + if stop.is_set(): + break + finally: + proc.terminate() + + +def main(): + def configure(p): + p.add_argument('--concurrency', type=int, default=12) + p.add_argument('--rounds', type=int, default=3) + p.add_argument('--output') + args = arguments('Concurrent upload memory experiment', configure) + check(args.concurrency >= 10, 'use at least 10 concurrent uploads') + c = Client(args.base_url, args.ca_file, timeout=60) + alice, bob, eve, chat = c.bootstrap() + data = png(1800, 1800, noise=True) + digest = hashlib.sha256(data).hexdigest() + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=load.png' + + idle, stop = {}, threading.Event() + t = threading.Thread(target=sample, args=(stop, idle)) + t.start() + time.sleep(4) + stop.set() + t.join() + + peaks, stop = {}, threading.Event() + t = threading.Thread(target=sample, args=(stop, peaks)) + t.start() + time.sleep(2) + barrier = threading.Barrier(args.concurrency) + results = [] + + def worker(_): + local = Client(args.base_url, args.ca_file, timeout=60) + barrier.wait() + out = [] + for _ in range(args.rounds): + try: + r = local.call('POST', path, 202, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment') + out.append(r['json']['sha256'] == digest) + except Failure: + out.append(False) + return out + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + for out in pool.map(worker, range(args.concurrency)): + results.extend(out) + elapsed = time.monotonic() - started + time.sleep(2) + stop.set() + t.join() + limit = 192 + report = { + 'file_bytes': len(data), + 'concurrent_uploads': args.concurrency, + 'rounds': args.rounds, + 'uploads_ok': sum(results), + 'uploads_failed': len(results) - sum(results), + 'elapsed_seconds': round(elapsed, 2), + 'api_memory_limit_mib': limit, + 'idle_memory_mib': {k: round(v, 1) for k, v in sorted(idle.items())}, + 'peak_memory_mib': {k: round(v, 1) for k, v in sorted(peaks.items())}, + 'buffering_all_files_would_need_mib': round(args.concurrency * len(data) / 1024 ** 2, 1), + } + text = json.dumps(report, indent=2) + if args.output: + Path(args.output).write_text(text + '\n') + print(text) + check(report['uploads_failed'] == 0, 'some uploads failed') + check(max(peaks.values()) < limit, 'peak memory reached the limit') + + +if __name__ == '__main__': + run(main) diff --git a/tests/variants.py b/tests/variants.py new file mode 100755 index 0000000..4c00905 --- /dev/null +++ b/tests/variants.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +import hashlib +import io +from PIL import Image +from client import Client, arguments, check, run +from stack import SAMPLE, upload, wait_status + +EXPECTED = {'thumbnail': ('image/jpeg', 'JPEG'), 'crop': ('image/jpeg', 'JPEG'), 'optimized': ('image/webp', 'WEBP')} + + +def encode(img, fmt, **kw): + buf = io.BytesIO() + img.save(buf, fmt, **kw) + return buf.getvalue() + + +def halves(w, h): + img = Image.new('RGB', (w, h), (0, 0, 255)) + img.paste((255, 0, 0), (0, 0, w // 2, h)) + return img + + +def process(c, alice, bob, chat, data, content_type='image/png'): + att = upload(c, alice, chat, data, content_type)['json'] + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + check(ready['status'] == 'ready' and ready['error_code'] is None, 'attachment failed: %s' % ready['error_code']) + check(sorted(v['name'] for v in ready['variants']) == sorted(EXPECTED), 'unexpected variant set') + original = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] + check(original == data and ready['sha256'] == hashlib.sha256(data).hexdigest(), 'original changed') + out = {} + for v in ready['variants']: + r = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + v['name'], actor=bob) + body = r['bytes'] + check(r['headers'].get_content_type() == v['content_type'] == EXPECTED[v['name']][0], v['name'] + ' content type') + check(len(body) == v['size_bytes'] and hashlib.sha256(body).hexdigest() == v['sha256'], v['name'] + ' checksum') + img = Image.open(io.BytesIO(body)) + check(img.format == EXPECTED[v['name']][1] and img.size == (v['width'], v['height']), v['name'] + ' real size/format') + check('exif' not in img.info, v['name'] + ' keeps EXIF') + img.load() + out[v['name']] = img.convert('RGB') + return att, out + + +def main(): + args = arguments('Thumbnail, crop and optimized variant rules') + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + + att, v = process(c, alice, bob, chat, SAMPLE) + check(v['thumbnail'].size == (256, 160) and v['crop'].size == (128, 128) and v['optimized'].size == (320, 200), 'fixture sizes') + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=bogus', 400, actor=bob) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 404, actor=eve) + + _, v = process(c, alice, bob, chat, encode(halves(2000, 1000), 'PNG')) + check(v['thumbnail'].size == (256, 128) and v['optimized'].size == (1280, 640) and v['crop'].size == (128, 128), 'large image sizes') + + _, v = process(c, alice, bob, chat, encode(halves(100, 50), 'PNG')) + check(v['thumbnail'].size == (100, 50) and v['optimized'].size == (100, 50), 'small image must not be upscaled') + check(v['crop'].size == (128, 128), 'crop must be 128x128') + + _, v = process(c, alice, bob, chat, encode(Image.new('RGBA', (300, 300), (0, 0, 0, 0)), 'PNG')) + for name, img in v.items(): + check(min(img.getpixel((img.width // 2, img.height // 2))) >= 250, name + ': alpha not composited on white') + + exif = Image.Exif() + exif[0x0112] = 6 + _, v = process(c, alice, bob, chat, encode(halves(300, 200), 'JPEG', quality=95, exif=exif.tobytes()), 'image/jpeg') + t = v['thumbnail'] + check(t.size == (170, 256) and v['optimized'].size == (200, 300), 'EXIF orientation not applied to dimensions') + top, bottom = t.getpixel((85, 20)), t.getpixel((85, 236)) + check(top[0] > 200 and top[2] < 60 and bottom[2] > 200 and bottom[0] < 60, 'EXIF rotation direction wrong') + print('PASS: thumbnail/crop/optimized sizes, formats, checksums, no upscale, alpha on white, EXIF orientation, no EXIF in variants') + + +if __name__ == '__main__': + run(main) diff --git a/tests/worker_load.py b/tests/worker_load.py new file mode 100755 index 0000000..97bdfd7 --- /dev/null +++ b/tests/worker_load.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +import json +import subprocess +import threading +import time +from pathlib import Path +from client import Client, arguments, check, run +from image_gen import png +from stack import SAMPLE, compose, job_row, psql, restore_worker, upload, wait_status + +LIMIT_MIB = 768 + + +def sample(stop, state): + while not stop.is_set(): + out = compose('exec', '-T', 'worker', 'grep', 'VmRSS', '/proc/1/status', check_rc=False).stdout.split() + if len(out) > 1: + state['rss'] = max(state['rss'], int(out[1]) / 1024) + n = psql("select count(*) from jobs where status = 'processing'") + state['processing'] = max(state['processing'], int(n or 0)) + time.sleep(0.2) + + +def main(): + def configure(p): + p.add_argument('--huge', type=int, default=6) + p.add_argument('--small', type=int, default=20) + p.add_argument('--output') + args = arguments('Worker concurrency, memory and time limits', configure) + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + concurrency = int(compose('exec', '-T', 'worker', 'printenv', 'WORKER_CONCURRENCY').stdout.strip()) + huge = png(6320, 6320) + + state, stop = {'rss': 0, 'processing': 0}, threading.Event() + t = threading.Thread(target=sample, args=(stop, state)) + t.start() + started = time.monotonic() + ids = [upload(c, alice, chat, huge)['json']['id'] for _ in range(args.huge)] + ids += [upload(c, alice, chat, SAMPLE)['json']['id'] for _ in range(args.small)] + accepted = time.monotonic() - started + results = [wait_status(c, bob, i, ('ready', 'failed'), 300)['status'] for i in ids] + elapsed = time.monotonic() - started + stop.set() + t.join() + container = compose('ps', '-q', 'worker').stdout.strip() + oom = subprocess.run(['docker', 'inspect', '-f', '{{.RestartCount}} {{.State.OOMKilled}}', container], + capture_output=True, text=True).stdout.split() + + compose('stop', 'worker') + slow = upload(c, alice, chat, huge)['json'] + compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'JOB_TIMEOUT_SECONDS': '1'}) + timed = wait_status(c, bob, slow['id'], ('ready', 'failed'), 120) + _, attempts, last_error = job_row(slow['id']) + restore_worker() + + report = { + 'huge_images': {'count': args.huge, 'pixels': 6320 * 6320, 'bytes': len(huge)}, + 'small_images': args.small, + 'worker_concurrency': concurrency, + 'max_processing_jobs_observed': state['processing'], + 'all_ready': results.count('ready') == len(results), + 'accept_seconds': round(accepted, 2), + 'all_ready_seconds': round(elapsed, 1), + 'worker_peak_rss_mib': round(state['rss'], 1), + 'worker_memory_limit_mib': LIMIT_MIB, + 'worker_restarts_oom': oom, + 'timeout_case': {'job_timeout_seconds': 1, 'status': timed['status'], 'error_code': timed['error_code'], + 'attempts': attempts, 'last_error': last_error}, + } + text = json.dumps(report, indent=2) + if args.output: + Path(args.output).write_text(text + '\n') + print(text) + check(report['all_ready'], 'some jobs failed under load') + check(state['processing'] <= concurrency, 'more jobs processing than WORKER_CONCURRENCY') + check(state['rss'] < LIMIT_MIB and oom == ['0', 'false'], 'worker reached memory limit') + check(timed['status'] == 'failed' and timed['error_code'] == 'retry_exhausted' and 'deadline' in last_error, + 'timeout must be a bounded transient failure') + print('PASS: bounded worker concurrency, memory below limit, job timeout enforced') + + +if __name__ == '__main__': + run(main) diff --git a/worker.go b/worker.go new file mode 100644 index 0000000..fb141f0 --- /dev/null +++ b/worker.go @@ -0,0 +1,489 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "math/rand/v2" + "os" + "os/signal" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +var errSource = errors.New("staging object missing or corrupted") + +type Worker struct { + db *pgxpool.Pool + store *Store + broker *Broker + log *slog.Logger + owner string + timeout time.Duration + lease time.Duration + backoffBase time.Duration + backoffMax time.Duration + maxBytes int64 + maxPixels int64 + fault string + busy atomic.Int64 +} + +type claimedJob struct { + id, attachmentID, owner string + attempt int + created time.Time + outcome string +} + +type source struct { + stagingKey, contentType, sha256 string + size int64 +} + +func finalKey(attachmentID string) string { + return objectPrefix + attachmentID + "/original" +} + +func variantKey(attachmentID string, spec variantSpec) string { + return fmt.Sprintf("%s%s/v%d/%s.%s", objectPrefix, attachmentID, pipelineVersion, spec.name, spec.ext) +} + +func (w *Worker) inject(point string) { + if w.fault == point { + w.log.Error("fault injected", "point", point) + os.Exit(3) + } +} + +func (w *Worker) backoff(attempt int) time.Duration { + d := w.backoffBase << (attempt - 1) + if d <= 0 || d > w.backoffMax { + d = w.backoffMax + } + return d/2 + rand.N(d/2+1) +} + +func (w *Worker) claim(ctx context.Context, jobID string) (*claimedJob, error) { + tx, err := w.db.Begin(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback(ctx) + job := &claimedJob{id: jobID, owner: w.owner} + err = tx.QueryRow(ctx, ` + update jobs set status = 'processing', lease_owner = $2, + lease_until = now() + make_interval(secs => $3), updated_at = now() + where id = $1::uuid and status = 'queued' and next_attempt_at <= now() + returning attachment_id::text, attempt, created_at`, jobID, w.owner, w.lease.Seconds()).Scan(&job.attachmentID, &job.attempt, &job.created) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if _, err := tx.Exec(ctx, `update attachments set status = 'processing', updated_at = now() where id = $1::uuid`, job.attachmentID); err != nil { + return nil, err + } + return job, tx.Commit(ctx) +} + +func (w *Worker) fetch(ctx context.Context, src source) ([]byte, error) { + obj, info, err := w.store.Get(ctx, src.stagingKey) + if err != nil { + if isNoSuchKey(err) { + return nil, errSource + } + return nil, err + } + defer obj.Close() + if info.Size != src.size || info.Size > w.maxBytes { + return nil, errSource + } + data, err := io.ReadAll(io.LimitReader(obj, w.maxBytes+1)) + if err != nil { + return nil, err + } + sum := sha256.Sum256(data) + if int64(len(data)) != src.size || hex.EncodeToString(sum[:]) != src.sha256 { + return nil, errSource + } + return data, nil +} + +func (w *Worker) execute(ctx context.Context, job *claimedJob) error { + ctx, cancel := context.WithTimeout(ctx, w.timeout) + defer cancel() + var src source + err := w.db.QueryRow(ctx, ` + select coalesce(staging_key, ''), content_type, sha256, size_bytes from attachments where id = $1::uuid`, + job.attachmentID).Scan(&src.stagingKey, &src.contentType, &src.sha256, &src.size) + if err != nil { + return err + } + data, err := w.fetch(ctx, src) + if err != nil { + return err + } + _, renderSpan := childSpan(ctx, "image render", trace.SpanKindInternal) + variants, err := renderVariants(ctx, data, w.maxPixels) + endSpan(renderSpan, err) + if err != nil { + return err + } + original := finalKey(job.attachmentID) + if err := w.store.Put(ctx, original, bytes.NewReader(data), int64(len(data)), src.contentType); err != nil { + return err + } + for _, v := range variants { + if err := w.store.Put(ctx, variantKey(job.attachmentID, v.spec), bytes.NewReader(v.data), int64(len(v.data)), v.spec.contentType); err != nil { + return err + } + } + w.inject("after_store") + + tx, err := w.db.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + tag, err := tx.Exec(ctx, ` + update jobs set status = 'ready', lease_owner = null, lease_until = null, last_error = null, updated_at = now() + where id = $1::uuid and status = 'processing' and lease_owner = $2 and attempt = $3`, job.id, job.owner, job.attempt) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + job.outcome = "lease_lost" + w.log.WarnContext(ctx, "lease lost, result discarded", "job_id", job.id, "attempt", job.attempt) + return nil + } + for _, v := range variants { + _, err := tx.Exec(ctx, ` + insert into attachment_variants(attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256) + values($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9) + on conflict (attachment_id, pipeline_version, name) do update + set object_key = excluded.object_key, content_type = excluded.content_type, size_bytes = excluded.size_bytes, + width = excluded.width, height = excluded.height, sha256 = excluded.sha256`, + job.attachmentID, pipelineVersion, v.spec.name, variantKey(job.attachmentID, v.spec), v.spec.contentType, + len(v.data), v.width, v.height, v.sha256) + if err != nil { + return err + } + } + if _, err := tx.Exec(ctx, ` + update attachments set status = 'ready', error_code = null, object_key = $2, updated_at = now() + where id = $1::uuid`, job.attachmentID, original); err != nil { + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + job.outcome = "ready" + jobTimeToReady.Record(ctx, time.Since(job.created).Seconds()) + w.log.InfoContext(ctx, "job ready", "job_id", job.id, "attachment_id", job.attachmentID, "attempt", job.attempt) + w.inject("before_ack") + if err := w.store.Remove(ctx, src.stagingKey); err != nil { + w.log.WarnContext(ctx, "staging cleanup deferred to janitor", "attachment_id", job.attachmentID, "err", err.Error()) + } + return nil +} + +func (w *Worker) settle(ctx context.Context, job *claimedJob, permanent bool, code, reason string) error { + tx, err := w.db.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + var maxAttempts int + if err := tx.QueryRow(ctx, `select max_attempts from jobs where id = $1::uuid`, job.id).Scan(&maxAttempts); err != nil { + return err + } + fence := ` where id = $1::uuid and status = 'processing' and lease_owner = $2 and attempt = $3` + if !permanent && job.attempt < maxAttempts { + delay := w.backoff(job.attempt) + tag, err := tx.Exec(ctx, ` + update jobs set status = 'queued', attempt = attempt + 1, next_attempt_at = now() + make_interval(secs => $4), + lease_owner = null, lease_until = null, last_error = $5, updated_at = now()`+fence, + job.id, job.owner, job.attempt, delay.Seconds(), reason) + if err != nil || tag.RowsAffected() == 0 { + return err + } + if _, err := tx.Exec(ctx, ` + insert into outbox(job_id, attempt, available_at, created_at) + values($1::uuid, $2, now() + make_interval(secs => $3), now())`, job.id, job.attempt+1, delay.Seconds()); err != nil { + return err + } + if _, err := tx.Exec(ctx, `update attachments set status = 'queued', updated_at = now() where id = $1::uuid`, job.attachmentID); err != nil { + return err + } + job.outcome = "retry" + w.log.WarnContext(ctx, "job retry scheduled", "job_id", job.id, "attempt", job.attempt, "next_attempt", job.attempt+1, + "delay_ms", delay.Milliseconds(), "reason", reason) + return tx.Commit(ctx) + } + if !permanent { + code = "retry_exhausted" + } + tag, err := tx.Exec(ctx, ` + update jobs set status = 'failed', lease_owner = null, lease_until = null, last_error = $4, updated_at = now()`+fence, + job.id, job.owner, job.attempt, reason) + if err != nil || tag.RowsAffected() == 0 { + return err + } + if _, err := tx.Exec(ctx, ` + update attachments set status = 'failed', error_code = $2, updated_at = now() where id = $1::uuid`, job.attachmentID, code); err != nil { + return err + } + job.outcome = "failed" + w.log.ErrorContext(ctx, "job failed", "job_id", job.id, "attempt", job.attempt, "error_code", code, "reason", reason) + return tx.Commit(ctx) +} + +func (w *Worker) handle(ctx context.Context, msg redis.XMessage) bool { + var m jobMessage + raw, _ := msg.Values["job"].(string) + if err := json.Unmarshal([]byte(raw), &m); err != nil || m.SchemaVersion != 1 || m.Type != jobType || !isUUID(m.JobID) { + w.log.Warn("invalid job message dropped", "stream_id", msg.ID) + return true + } + ctx = extractTrace(ctx, traceCarrier{m.Traceparent, m.Tracestate}) + ctx, span := tracer.Start(ctx, jobStream+" process", trace.WithSpanKind(trace.SpanKindConsumer), trace.WithAttributes( + attribute.String("messaging.system", "redis"), + attribute.String("messaging.destination.name", jobStream), + attribute.String("messaging.operation.type", "process"), + attribute.String("job.id", m.JobID), + attribute.String("attachment.id", m.AttachmentID), + attribute.Int("job.attempt", m.Attempt), + )) + started := time.Now() + w.busy.Add(1) + defer w.busy.Add(-1) + job, err := w.claim(ctx, m.JobID) + if err != nil { + w.log.WarnContext(ctx, "claim failed", "job_id", m.JobID, "err", err.Error()) + endSpan(span, err) + return false + } + if job == nil { + var status string + var attempt int + _ = w.db.QueryRow(ctx, `select status, attempt from jobs where id = $1::uuid`, m.JobID).Scan(&status, &attempt) + w.log.InfoContext(ctx, "job delivery skipped", "job_id", m.JobID, "message_attempt", m.Attempt, "status", status, "attempt", attempt) + span.SetAttributes(attribute.String("job.outcome", "skipped")) + jobsProcessed.Add(ctx, 1, metric.WithAttributes(attribute.String("job.type", jobType), attribute.String("outcome", "skipped"))) + span.End() + return true + } + err = w.execute(ctx, job) + defer func() { + out := metric.WithAttributes(attribute.String("job.type", jobType), attribute.String("outcome", job.outcome)) + jobsProcessed.Add(ctx, 1, out) + jobDuration.Record(ctx, time.Since(started).Seconds(), out) + span.SetAttributes(attribute.String("job.outcome", job.outcome)) + endSpan(span, err) + }() + if err == nil { + return true + } + permanent, code := false, "processing_failed" + switch { + case errors.Is(err, errInvalidImage): + permanent, code = true, "invalid_image" + case errors.Is(err, errSource): + permanent = true + } + cause := err + if err := w.settle(ctx, job, permanent, code, cause.Error()); err != nil { + w.log.WarnContext(ctx, "settle failed", "job_id", job.id, "err", err.Error()) + job.outcome = "error" + return false + } + return true +} + +func (w *Worker) read(ctx context.Context, consumer, id string, block time.Duration) ([]redis.XMessage, error) { + res, err := w.broker.rdb.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: jobGroup, Consumer: consumer, Streams: []string{jobStream, id}, Count: 1, Block: block, + }).Result() + if len(res) == 0 { + return nil, err + } + return res[0].Messages, err +} + +func (w *Worker) consume(ctx context.Context, consumer string) { + pending := true + nextClaim := time.Now().Add(w.lease) + for ctx.Err() == nil { + var msgs []redis.XMessage + var err error + switch { + case pending: + msgs, err = w.read(ctx, consumer, "0", -1) + pending = len(msgs) > 0 + case time.Now().After(nextClaim): + msgs, _, err = w.broker.rdb.XAutoClaim(ctx, &redis.XAutoClaimArgs{ + Stream: jobStream, Group: jobGroup, Consumer: consumer, MinIdle: w.lease, Start: "0-0", Count: 1, + }).Result() + nextClaim = time.Now().Add(15 * time.Second) + default: + msgs, err = w.read(ctx, consumer, ">", 5*time.Second) + } + if err != nil && !errors.Is(err, redis.Nil) { + if ctx.Err() != nil { + return + } + if strings.Contains(err.Error(), "NOGROUP") { + _ = w.broker.ensureGroup(ctx) + } + w.log.Warn("broker read failed", "consumer", consumer, "err", err.Error()) + time.Sleep(2 * time.Second) + continue + } + for _, msg := range msgs { + if !w.handle(context.WithoutCancel(ctx), msg) { + pending = false + continue + } + if err := w.broker.rdb.XAck(context.WithoutCancel(ctx), jobStream, jobGroup, msg.ID).Err(); err != nil { + w.log.Warn("ack failed", "stream_id", msg.ID, "err", err.Error()) + } + } + } +} + +func (w *Worker) reap(ctx context.Context) error { + rows, err := w.db.Query(ctx, ` + select id::text, attachment_id::text, attempt, coalesce(lease_owner, '') + from jobs where status = 'processing' and lease_until < now() limit 50`) + if err != nil { + return err + } + jobs, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (*claimedJob, error) { + j := &claimedJob{} + return j, row.Scan(&j.id, &j.attachmentID, &j.attempt, &j.owner) + }) + if err != nil { + return err + } + for _, j := range jobs { + w.log.Warn("lease expired", "job_id", j.id, "attempt", j.attempt) + if err := w.settle(ctx, j, false, "", "lease expired"); err != nil { + return err + } + jobsProcessed.Add(ctx, 1, metric.WithAttributes(attribute.String("job.type", jobType), attribute.String("outcome", j.outcome))) + } + return nil +} + +func runWorker(db *pgxpool.Pool, store *Store, broker *Broker, log *slog.Logger, w *Worker, concurrency int) { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + for { + err := broker.ensureGroup(ctx) + if err == nil { + break + } + log.Warn("broker not ready", "err", err.Error()) + select { + case <-ctx.Done(): + return + case <-time.After(2 * time.Second): + } + } + host, _ := os.Hostname() + var wg sync.WaitGroup + wg.Go(func() { + for ctx.Err() == nil { + if err := w.reap(ctx); err != nil && ctx.Err() == nil { + log.Warn("lease reaper failed", "err", err.Error()) + } + select { + case <-ctx.Done(): + case <-time.After(5 * time.Second): + } + } + }) + for i := range concurrency { + consumer := fmt.Sprintf("%s-%d", host, i) + wg.Go(func() { w.consume(ctx, consumer) }) + } + registerWorkerMetrics(w, concurrency) + log.Info("worker started", "owner", w.owner, "concurrency", concurrency, "lease_s", w.lease.Seconds()) + <-ctx.Done() + log.Info("worker stopping") + wg.Wait() +} + +func replayJob(ctx context.Context, db *pgxpool.Pool, attachmentID string, maxAttempts int) (string, error) { + tx, err := db.Begin(ctx) + if err != nil { + return "", err + } + defer tx.Rollback(ctx) + var jobID, status string + var attempt int + var lastErr *string + err = tx.QueryRow(ctx, `select id::text, status, attempt, last_error from jobs where attachment_id = $1::uuid for update`, + attachmentID).Scan(&jobID, &status, &attempt, &lastErr) + if errors.Is(err, pgx.ErrNoRows) { + return "not_found", nil + } + if err != nil { + return "", err + } + if status != "failed" { + return "skipped_" + status, nil + } + if _, err := tx.Exec(ctx, `insert into job_replays(job_id, previous_error, replayed_at) values($1::uuid, $2, now())`, jobID, lastErr); err != nil { + return "", err + } + if _, err := tx.Exec(ctx, ` + update jobs set status = 'queued', attempt = attempt + 1, max_attempts = attempt + $2, next_attempt_at = now(), + last_error = null, updated_at = now() + where id = $1::uuid`, jobID, maxAttempts); err != nil { + return "", err + } + if _, err := tx.Exec(ctx, `insert into outbox(job_id, attempt, available_at, created_at) values($1::uuid, $2, now(), now())`, jobID, attempt+1); err != nil { + return "", err + } + if _, err := tx.Exec(ctx, `update attachments set status = 'queued', error_code = null, updated_at = now() where id = $1::uuid`, attachmentID); err != nil { + return "", err + } + return "queued", tx.Commit(ctx) +} + +func listFailed(ctx context.Context, db *pgxpool.Pool) ([]map[string]any, error) { + rows, err := db.Query(ctx, ` + select j.attachment_id::text, j.id::text, j.attempt, coalesce(a.error_code, ''), coalesce(j.last_error, ''), j.updated_at + from jobs j join attachments a on a.id = j.attachment_id + where j.status = 'failed' order by j.updated_at`) + if err != nil { + return nil, err + } + return pgx.CollectRows(rows, func(row pgx.CollectableRow) (map[string]any, error) { + var att, job, code, reason string + var attempt int + var updated time.Time + err := row.Scan(&att, &job, &attempt, &code, &reason, &updated) + return map[string]any{"attachment_id": att, "job_id": job, "attempt": attempt, "error_code": code, + "last_error": reason, "failed_at": formatTime(updated)}, err + }) +} -- 2.54.0