diff --git a/.env.example b/.env.example index 2b32655..1739116 100644 --- a/.env.example +++ b/.env.example @@ -28,3 +28,7 @@ OTEL_SERVICE_NAME=messenger-api OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +CLEANUP_GRACE_SECONDS=600 +CLEANUP_INTERVAL_SECONDS=300 +JOB_BACKOFF_BASE_MS=1000 +JOB_BACKOFF_MAX_MS=30000 diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..49405e7 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,37 @@ +# Изменения lab6 относительно lab5 + +Бизнес-API не менялся. Контракт OpenAPI, схема `jobs.schema.json` и публичные тесты остались как в skeleton. + +## Приложение + +- `telemetry.go`: + - OTLP/HTTP экспорт трасс, метрик и логов в Collector с ограниченными очередями и повторами; + - `slog`-обработчик пишет JSON в stdout и тот же лог в OTLP; в stdout добавляются `service.name`, `event`, `trace_id`, `span_id`; + - инструменты метрик, process CPU/RSS, DB pool, очередь, saturation worker; + - трассировщик запросов pgx и вспомогательные функции spans и W3C trace context. +- `handlers.go`: middleware создаёт server span, извлекает входящий `traceparent`, считает `http.server.requests` и `http.server.request.duration` с route template и классом статуса. Health-запросы не создают spans и метрики. +- `broker.go`: сообщение получает поля `traceparent` и `tracestate` (разрешены `jobs.schema.json`); relay создаёт producer span `attachments.jobs publish` как потомка запроса загрузки. +- `worker.go`: consumer span `attachments.jobs process` продолжает trace из сообщения; метрики `jobs.processed`, `job.processing.duration`, `job.time_to_ready`, `worker.busy`; логи привязаны к trace. +- `attachments.go`: контекст trace сохраняется в `jobs.traceparent` и `jobs.tracestate` в той же транзакции, что и задача. Повторы, reaper и replay остаются в исходном trace. +- `s3.go`: spans `s3 put`, `s3 get`, `s3 remove` внутри существующих trace. +- `db.go`: трассировка запросов и метрики пула. +- `main.go`: телеметрия включается только для `api` и `worker`; метрики очереди регистрирует API. +- `migrations/0005_trace_context.sql`: колонки `traceparent`, `tracestate` в `jobs`, индекс очереди. +- Зависимости: `go.opentelemetry.io/otel` и OTLP/HTTP экспортёры, мост `otelslog`. + +## Инфраструктура + +- `compose.yaml`: сервисы `otel-collector`, `prometheus`, `loki`, `tempo`, `grafana` с закреплёнными digest, лимитами, healthcheck и volumes; сеть `telemetry`; образ приложения `mtusi-messenger:lab6`; OTEL-переменные в `api` и `worker`. +- `infra/otel/collector.yaml`: receivers OTLP, `memory_limiter`, `batch`, ограниченные очереди и повторы экспортёров, три pipeline. +- `infra/prometheus/`: конфигурация с OTLP receiver и alert rules `ApiHighErrorRate`, `QueueLagGrowing`. +- `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml`: хранилища логов и трасс на локальных volumes, retention 7 суток. +- `infra/grafana/`: provisioning datasources (с переходом лог → trace и trace → лог) и dashboard `messenger.json`. +- `infra/runbook.md`: runbook для alerts. +- `scripts/mksecrets.sh`: пароль Grafana. `scripts/restart.sh`: перезапускает хранилища телеметрии и Collector. +- `.env.example`: переменные очистки и backoff, которые читает compose. + +## Тесты + +Новые: `tests/obs.py` (запросы к Prometheus, Loki, Tempo), `tests/observability.py` (сигналы, цепочка trace, корреляция логов, retry в одном trace, отсутствие секретов), `tests/incidents.py` (остановка worker, блокировка БД, остановка хранилищ телеметрии), `tests/dashboard.py` (provisioning и запросы dashboard через Grafana). + +Перенесены из lab5 без изменений: `attachments.py`, `s3_checks.py`, `idempotency.py`, `session_revocation.py`, `variants.py`, `queue_recovery.py`, `faults.py`, `worker_load.py`, `upload_memory.py`, `scenario.py`, `stack.py`, `image_gen.py`. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..94f0316 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM golang:1.26-alpine@sha256:c95332c2af86b6d89b91bd0500f4b9529ccbd090a0d1855c6d1ceaa142ae8615 AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY *.go ./ +COPY migrations ./migrations +RUN CGO_ENABLED=0 GOOS=linux go build -tags nodynamic -trimpath -ldflags="-s -w" -o /out/messenger . + +FROM alpine:3.21@sha256:ce64758a109eb420d874a118f87920e625e12d3634e03b4a5573fd9f6e5d3507 +RUN adduser -D -u 10001 app +WORKDIR /app +COPY --from=build /out/messenger /app/messenger +USER 10001 +EXPOSE 8080 +ENV APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +STOPSIGNAL SIGTERM +HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \ + CMD wget -qO- http://127.0.0.1:8080/health/ready >/dev/null || exit 1 +ENTRYPOINT ["/app/messenger"] diff --git a/REPORT.md b/REPORT.md index b97ae42..184c445 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,230 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Березин Герман / БВТ2304 +- Целевая оценка: 4 +- Commit/PR: после публикации +- Использованные источники, библиотеки и помощь ИИ: Go 1.26, OpenTelemetry Go SDK 1.47 (`otel`, `sdk`, `sdk/metric`, `sdk/log`, экспортёры OTLP/HTTP, мост `otelslog` 0.21), pgx, go-redis, minio-go; OpenTelemetry Collector contrib 0.116.1, Prometheus 3.1.0, Loki 3.3.2, Tempo 2.7.0, Grafana 11.3.0; документация OTel Collector configuration, OTel propagation, W3C Trace Context, Prometheus histograms, Grafana provisioning из README; код lab5 перенесён за основу; ИИ-ассистент использован при разработке ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker 29 с Compose v2 и Python 3.9+. Для `tests/variants.py` нужен Pillow. + +```sh +cp .env.example .env +sh scripts/mksecrets.sh +docker compose --env-file .env up -d --build --wait +make check && make test +``` + +`mksecrets.sh` создаёт секреты в `./secrets` (вне Git), в том числе пароль Grafana: `cat secrets/grafana_admin_password`, пользователь `admin`. Существующие файлы не перезаписываются. + +Порядок старта: postgres, valkey, broker, s3, prometheus, loki, tempo → `migrate` → `s3-init`, otel-collector, grafana → `api` × 2, `worker`, `janitor` → `lb`. Публичный адрес API: `http://localhost:8080`. Интерфейсы на `127.0.0.1`: Grafana `:3000`, Prometheus `:9090` (alerts: `/alerts`), Loki `:3100`, Tempo `:3200`. Dashboard и datasources подключаются автоматически из `infra/grafana/`, вручную ничего набирать не нужно. После первого старта S3 принимает запись через 10–30 с. + +Остановка без потери данных: `docker compose --env-file .env stop`. Намеренное удаление учебных данных: `docker compose --env-file .env down -v`. + +Проверки (стенд запущен): + +```sh +python3 tests/dashboard.py +python3 tests/observability.py +python3 tests/incidents.py --only worker +python3 tests/incidents.py --only database +python3 tests/incidents.py --only storage +python3 tests/telemetry_persistence.py +python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/scenario.py --mode failover-each --output evidence/failover-each.json +python3 tests/scenario.py --mode rolling --output evidence/rolling.json +python3 tests/variants.py +python3 tests/queue_recovery.py +python3 tests/faults.py +python3 tests/worker_load.py --output evidence/worker-load.json +python3 tests/attachments.py +python3 tests/s3_checks.py +python3 tests/upload_memory.py --output evidence/upload-memory.json +python3 tests/idempotency.py +python3 tests/session_revocation.py +``` + +`incidents.py`, `observability.py`, `faults.py`, `queue_recovery.py`, `worker_load.py` сами останавливают и пересоздают сервисы и возвращают их. После `make test-failover` реплику возвращают командой `docker compose --env-file .env up -d --wait api`. Операторские команды worker и разбор alerts: `infra/runbook.md`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +клиент → HAProxy → api ×2 ──OTLP──┐ + │ trace context в jobs.traceparent → outbox → сообщение broker + ▼ │ + PostgreSQL / S3 / broker ▼ + worker ──OTLP──┤ + ▼ + otel-collector (memory_limiter, batch, очереди) + метрики ─→ Prometheus логи ─→ Loki трассы ─→ Tempo + ▼ + Grafana (datasources и dashboard из Git) +``` + +Бизнес-API, PostgreSQL, S3, broker, outbox, worker и retry перенесены из lab5 без изменения смысла (подробности в `CHANGES.md`). Изменено: добавлена телеметрия, миграция `0005_trace_context.sql`, сервисы наблюдаемости. + +| Роль | Сервис | Образ | Готовность | Volume | Сеть | Назначение | +| --- | --- | --- | --- | --- | --- | --- | +| collector | `otel-collector` | otel/opentelemetry-collector-contrib:0.116.1@sha256:d0ebf652… | запуск после готовности хранилищ | нет | backend, telemetry | принимает OTLP, ограничивает память, батчит, раздаёт сигналы | +| метрики | `prometheus` | prom/prometheus:v3.1.0@sha256:6559acbd… | `/-/ready` | prometheus_data | telemetry, 127.0.0.1:9090 | хранит метрики (OTLP receiver), считает alert rules | +| логи | `loki` | grafana/loki:3.3.2@sha256:8af2de1a… | `/ready` | loki_data | telemetry, 127.0.0.1:3100 | хранит структурные логи, ищет по trace_id | +| трассы | `tempo` | grafana/tempo:2.7.0@sha256:12e904cc… | `/ready` | tempo_data | telemetry, 127.0.0.1:3200 | хранит трассы, поиск TraceQL | +| интерфейс | `grafana` | grafana/grafana:11.3.0@sha256:a0f88123… | `/api/health` | grafana_data | telemetry, 127.0.0.1:3000 | dashboard и запросы к трём источникам, сам данные не хранит | +| api ×2, worker, postgres, valkey, broker, s3, lb | как в lab5 | образ приложения `mtusi-messenger:lab6` | как в lab5 | как в lab5 | backend, storage | см. REPORT lab5 | + +Bootstrap: `migrate` применяет `migrations/*.sql`; provisioning Grafana читается при старте из `infra/grafana/provisioning`; Prometheus, Loki и Tempo стартуют из конфигов в `infra/`. Повторный запуск `up -d` безопасен. Пароль Grafana передаётся файлом secret (`GF_SECURITY_ADMIN_PASSWORD__FILE`). + +- **Collector** (`infra/otel/collector.yaml`): receivers OTLP gRPC/HTTP; processors `memory_limiter` (проверка 1 с, лимит 160 MiB, spike 40 MiB; контейнер 256 MiB) и `batch`; exporters `otlphttp` в Prometheus (`/api/v1/otlp`) и Loki (`/otlp`), `otlp` gRPC в Tempo. У каждого экспортёра очередь `sending_queue` на 100 батчей и повтор до 20 с (`retry_on_failure`), после чего данные отбрасываются. Collector не хранит данные на диске. +- **Приложение.** SDK экспортирует по OTLP/HTTP: батч трасс (очередь 2048 spans, таймаут 5 с), метрики раз в 10 с, батч логов (очередь 2048, интервал 1 с); повтор каждого экспорта не дольше 8 с. Недоступный Collector не блокирует запросы: полная очередь отбрасывает записи, ошибки экспорта считает `telemetry.export.errors`. +- **Распространение контекста.** Входящий `traceparent` запроса извлекается. Upload сохраняет контекст server span в `jobs.traceparent`/`tracestate` в той же транзакции, что и задачу. Relay берёт его из строки задачи, создаёт producer span `attachments.jobs publish` (родитель — запрос) и кладёт новый `traceparent` в сообщение (поля разрешены `jobs.schema.json`). Worker извлекает контекст и создаёт consumer span `attachments.jobs process` с родителем producer span. Повторы, reaper и `replay` публикуют сообщение из той же строки, поэтому все попытки лежат в одном trace, у каждой свой producer и consumer span. +- **Spans:** server span на запрос (имя `METHOD route template`), `db <операция>` на каждый запрос pgx (текст SQL без параметров), `s3 put/get/remove`, `image render`. Spans создаются только внутри существующего trace: фоновые опросы (relay, reaper, чтение stream) не порождают отдельных trace. +- **Логи.** JSON в stdout и тот же лог по OTLP в Loki. Поля: время, уровень, `service.name`, `event`, `trace_id`, `span_id` при активном span, технические поля события. Resource attributes: `service.name` (`messenger-api` / `messenger-worker`), `service.version`, `service.instance.id` (id контейнера), `deployment.environment.name=lab`. +- **Что не попадает в телеметрию.** Заголовок Authorization, токены, пароли, тела сообщений, содержимое изображений, URL с параметрами (в spans только `http.route` — шаблон), значения параметров SQL. В labels метрик нет user_id, chat_id, message_id, attachment_id, request_id, filename, raw URL. В логах и spans есть технические `request_id`, `job_id`, `attachment_id` (синтетические учебные данные): по ним находится нужный trace. +- **Хранилища и сохранность.** Prometheus, Loki и Tempo хранят данные 7 суток на своих volumes (Loki с WAL и сбросом при остановке). Историю после перезапуска проверяет `tests/telemetry_persistence.py`. + +### Соответствие .env.example + +| Переменная | Где используется | +| --- | --- | +| `OTEL_SERVICE_NAME` | api: `messenger-api`; worker: `messenger-worker` (задаётся в compose) | +| `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_PROTOCOL` | compose, api и worker; SDK читает стандартные переменные | +| `OTEL_RESOURCE_ATTRIBUTES` | `deployment.environment.name`, SDK читает стандартную переменную | +| остальные | как в lab5 (`API_REPLICAS`, `S3_*`, `WORKER_CONCURRENCY`, `JOB_*`, `CLEANUP_*`, `STAGING_RETENTION_SECONDS`, `*_FILE`) | + +### Сигналы и соответствие контракту + +Имена OTel и их вид в Prometheus (единицы в имени добавляет трансляция OTLP → Prometheus). Для каждого сигнала один инструмент, дублей нет. + +| Контракт | Инструмент OTel (единицы) | В Prometheus | Измерения | +| --- | --- | --- | --- | +| HTTP requests | `http.server.requests` (counter) | `http_server_requests_total` | method, route template, класс статуса (`2xx`…) | +| HTTP duration | `http.server.request.duration` (histogram, с) | `http_server_request_duration_seconds_bucket` | method, route, класс статуса; buckets 0.005–10 с | +| Jobs processed / failed | `jobs.processed` (counter) | `jobs_processed_total` | job.type, outcome: `ready`, `retry`, `failed`, `skipped`, `lease_lost` | +| Job duration | `job.processing.duration` (histogram, с) | `job_processing_duration_seconds_bucket` | job.type, outcome | +| Time to ready | `job.time_to_ready` (histogram, с) | `job_time_to_ready_seconds_bucket` | от durable acceptance (`jobs.created_at`) до COMMIT ready | +| Queue depth / oldest age | `queue.depth` ({job}), `queue.oldest_job_age` (с) | `queue_depth`, `queue_oldest_job_age_seconds` | queue; из PostgreSQL (`jobs` в статусе `queued`), возраст от `next_attempt_at` | +| DB pool | `db.pool.connections`, `db.pool.max_connections`, `db.pool.empty_acquires`, `db.pool.acquire.wait` (с) | `db_pool_connections{state}`, `db_pool_max_connections`, `db_pool_empty_acquires_total`, `db_pool_acquire_wait_seconds_total` | state `used`/`idle` | +| Worker saturation | `worker.busy`, `worker.capacity` | `worker_busy`, `worker_capacity` | | +| CPU / RAM | `process.cpu.time` (с), `process.memory.usage` (байт) | `process_cpu_time_seconds_total`, `process_memory_usage_bytes` | процесс (api, worker) | +| Потери телеметрии | `telemetry.export.errors`; метрики Collector `otelcol_exporter_*` (scrape Prometheus) | `telemetry_export_errors_total`, `otelcol_exporter_send_failed_*`, `otelcol_exporter_queue_size` | | +| Логи, трассы | OTLP logs, OTLP traces | Loki (`service_name`, structured metadata), Tempo | `trace_id`, `span_id` | + +Метрики очереди публикует каждая API-реплика (одно значение из PostgreSQL), на dashboard и в alert применяется `max`, чтобы данные не пропадали при остановке worker. + +### SLI и SLO + +| Цель | SLI | SLO | +| --- | --- | --- | +| Доступность API | доля бизнес-запросов без ответа 5xx: `1 − sum(rate(http_server_requests_total{status_class="5xx"})) / sum(rate(http_server_requests_total))`; health-запросы не учитываются, 4xx (ошибки клиента, например неверный пароль) не считаются отказом | 99% за скользящие 30 суток | +| Время до ready | доля вложений с `job_time_to_ready_seconds ≤ 10`: `increase(…_bucket{le="10"}) / increase(…_count)` | 95% за 30 суток | + +Alert rules (`infra/prometheus/rules/alerts.yml`, runbook `infra/runbook.md`) — быстрые проверки расхода бюджета на коротких окнах, чтобы воспроизводить их на стенде: + +| Alert | Условие | Окно / `for` | Что означает | +| --- | --- | --- | --- | +| `ApiHighErrorRate` | доля 5xx > 5% | 1 мин / 20 с | бюджет доступности расходуется быстрее допустимого | +| `QueueLagGrowing` | `max(queue_oldest_job_age_seconds) > 20` | — / 20 с | worker не успевает или остановлен, SLO времени до ready под угрозой | + +Окно 30 суток на стенде не накапливается: показаны SLI за выбранный интервал dashboard и alerts, проверенные на коротком окне. Burn-rate с несколькими окнами не реализован (это трек 5). + +### Dashboard «Messenger observability» + +Файл `infra/grafana/dashboards/messenger.json`, импортируется provisioning'ом. Все 28 панелей и 34 запроса выполняются без ошибок (`tests/dashboard.py`). Ключевые панели: + +| Панель | Запрос | Единица | +| --- | --- | --- | +| RPS по маршрутам | `sum by (http_route) (rate(http_server_requests_total[$__rate_interval]))` | запросов/с | +| Доля ошибок | `(sum(rate(…{http_response_status_class="5xx"}[$__rate_interval])) or vector(0)) / sum(rate(http_server_requests_total[$__rate_interval]))` | доля (0–1) | +| Latency p50/p95/p99 | `histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket[$__rate_interval])))` | с | +| Jobs по outcome | `sum by (outcome) (rate(jobs_processed_total[$__rate_interval]))` | задач/с | +| Job duration p50/p95/p99 | `histogram_quantile(0.95, sum by (le) (rate(job_processing_duration_seconds_bucket[…])))` | с | +| Time to ready p50/p95/p99 | `histogram_quantile(…, sum by (le) (rate(job_time_to_ready_seconds_bucket[…])))` | с | +| Queue depth | `max by (queue) (queue_depth)` | задач | +| Oldest queued job age | `max by (queue) (queue_oldest_job_age_seconds)` | с | +| Worker saturation | `sum(worker_busy)`, `sum(worker_capacity)` | слотов | +| DB pool | `sum by (instance) (db_pool_connections{state="used",job="messenger-api"})`, `max(db_pool_max_connections)` | соединений | +| DB acquire wait | `sum by (job) (rate(db_pool_acquire_wait_seconds_total[…]))`, `rate(db_pool_empty_acquires_total[…])` | с/с, ожиданий/с | +| CPU, RSS | `rate(process_cpu_time_seconds_total[…])`, `process_memory_usage_bytes` | ядра, байты | +| SLI доступности, SLI ready ≤ 10 с | см. таблицу SLI | доля | +| Collector: очередь, потери, память | `otelcol_exporter_queue_size`, `rate(otelcol_exporter_send_failed_*[…])`, `otelcol_process_memory_rss` | батчей, записей/с, байты | +| Логи warn/error, трассы с ошибкой | Loki `{service_name=~"messenger-.*"} \| detected_level=~"warn\|error"`, TraceQL `{ status = error }` | | + +Percentiles берутся из histogram buckets суммой по всем репликам (`sum by (le)`), средние p95 реплик не используются. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke при включённой телеметрии | `make check && make test` | PASS | evidence/smoke.txt | +| Минимум lab5 сохранён | `tests/attachments.py`, `s3_checks.py`, `idempotency.py`, `session_revocation.py`, `variants.py`, `queue_recovery.py`, `faults.py`, `worker_load.py`, `upload_memory.py` | PASS; `s3_checks.py` допускает публикацию портов только на 127.0.0.1 и только у lb, Grafana, Prometheus, Loki, Tempo | evidence/attachments.txt, s3-checks.txt, idempotency.txt, session-revocation.txt, variants.txt, queue-recovery.txt, faults.txt, worker-load.json, upload-memory.json | +| Restart всех stores и телеметрии с volumes | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; времена запуска postgres, valkey, broker, s3, prometheus, loki, tempo, collector, grafana, api, lb, janitor, worker до и после | evidence/persistence.txt | +| Failover API-реплик, rolling update | `make test-failover`, `tests/scenario.py` | PASS для обеих реплик, 0 неуспешных попыток в публичном тесте, запись восстанавливается за 3.5 с | evidence/failover-1.txt, failover-2.txt, failover-each.json, rolling.json | +| Нагрузка | `tests/load.py --duration 30 --concurrency 4` | 11 666 успешных запросов, 387 RPS, 0 ошибок, p50/p95/p99 5.8 / 44.8 / 55.5 мс | evidence/load.json | +| Метрики, логи, трассы API и worker; низкая кардинальность; нет секретов | `tests/observability.py` | 13 сигналов есть; в labels нет user_id/chat_id/message_id/attachment_id/request_id; route — шаблон; тело сообщения и токен отсутствуют в Loki и Tempo | evidence/observability.txt | +| Запрос → trace → лог; job → worker trace → лог | `tests/observability.py` | по `request_id` лог API содержит `trace_id`/`span_id`; в Tempo trace из 30 spans; лог worker `job ready` с тем же `trace_id` | evidence/observability.txt | +| W3C trace context API → broker → worker | `tests/observability.py` | server → producer (родитель — запрос) → consumer (родитель — producer) → `s3 get`, `image render`, 4×`s3 put`, `db COMMIT`, `s3 remove` в одном trace | evidence/observability.txt | +| Цепочка по одному upload включая retry | `tests/observability.py` | S3 остановлен: 5 неуспешных попыток; после запуска S3 и `replay` попытка 6 → ready; 6 consumer и 6 producer spans в одном trace | evidence/observability.txt | +| Provisioning, импорт dashboard | `tests/dashboard.py` | 3 datasources и dashboard из Git, 28 панелей, 34 запроса без ошибок | evidence/dashboard.txt | +| Percentiles из histogram, доля ошибок, queue age, DB pool, saturation worker | dashboard (запросы выше) | панели показывают перечисленные величины | раздел «Dashboard», evidence/dashboard.txt | +| SLI/SLO и alerts, проверка искусственным отказом | инциденты 1 и 2 | firing, затем resolved | evidence/incident-worker.txt, incident-database.txt | +| Collector memory_limiter/batch, ограниченная буферизация, остановка хранилищ | `tests/incidents.py --only storage` | Prometheus, Loki, Tempo остановлены ≈ 80 с под нагрузкой: 942 сообщения 201 и 99 загрузок без 5xx; 3/3 вложения ready; RSS Collector 64 → 71 MiB; потери посчитаны | evidence/incident-storage.txt | +| История телеметрии на volumes | `tests/telemetry_persistence.py` | после перезапуска Prometheus, Loki, Tempo trace, его лог и метрики на месте | evidence/telemetry-persistence.txt | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Стенд: один хост, Docker 29, лимиты контейнеров из `compose.yaml`. Нагрузка — closed-loop клиент (3–4 потока, `Idempotency-Key`), в инцидентах плюс загрузка PNG раз в секунду. Числа ниже взяты из файлов evidence. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +### Инцидент 1. Остановлен worker + +Действие: `docker compose stop worker`, нагрузка загрузками. + +Диагностика только по сохранённым данным: +1. Alert `QueueLagGrowing` перешёл в firing через 60 с, возраст старейшей задачи 58 с, `queue_depth` 57. +2. Панели «Jobs per second by outcome» и «Worker saturation»: `jobs_processed_total` не растёт (0 задач/с), `worker_busy` равен 0. +3. Loki: за последнюю минуту нет ни одной записи `job ready` worker. +4. Tempo: trace свежей загрузки содержит server span API и producer span `attachments.jobs publish` и не содержит consumer span. Задача опубликована, её никто не забирает. + +Вывод: причина в остановленном worker, а не в API или брокере. Восстановление: `up -d worker`; alert разрешился через 20 с, очередь опустела за 21 с, потерь нет. + +### Инцидент 2. Заблокированная БД + +Действие: сессия psql держит `lock table messages in access exclusive mode` 45 с, нагрузка сообщениями. + +Диагностика: +1. `ApiHighErrorRate` в firing через 52 с, доля 5xx 0.47. +2. Метрики по маршрутам: 5xx только у `/api/v1/chats/{chat_id}/messages`. p99 ≈ 4.9 с. +3. Tempo, `{ status = error }`: server span 3006 мс, единственный долгий дочерний span `db INSERT` 2997 мс (≈ 100%) со статусом error. Время тратится в запросе к PostgreSQL (таймаут 3 с в `dbCtx`), а не в обработчике. +4. DB pool: ожидания соединения нет (0.01 с/с), пул не исчерпан. Значит причина не в нехватке соединений, а в одном долгом запросе. + +Вывод: причина в блокировке таблицы `messages` в PostgreSQL. Медленный обработчик исключён: у него нет собственного долгого этапа. Восстановление: блокировка снята, alert разрешился через 40 с; за время опыта 52 запроса получили 503, 164 — 201. + +### Инцидент 3. Недоступны хранилища телеметрии + +Prometheus, Loki, Tempo остановлены на ≈ 80 с под нагрузкой. API без ошибок (942 × 201, 99 загрузок), 3 новых вложения ready, сообщение создано. Collector не перезапускался и не убит по памяти, RSS 64 → 71 MiB при лимите контейнера 256 MiB. Потери посчитаны: `otelcol_exporter_send_failed_*` вырос на 2443 spans, 254 лога, 456 точек метрик. Данные за время недоступности потеряны (очередь 100 батчей и повтор до 20 с их не удерживают), данные до остановки сохранены, новые логи и метрики после запуска хранятся. + +Порог `memory_limiter` (160 MiB) в этом опыте не достигался: ограничение обеспечили ограниченные очереди и отбрасывание. + +### Прочее + +| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Отказ каждой API-реплики под нагрузкой | запись продолжается | 7 и 2 неуспешные попытки, 280 и 289 RPS против 448 до отказа | возврат реплики ≈ 6.8 с, потерь нет | +| Rolling update | без простоя | 2 неуспешные попытки, 462 RPS | старые реплики не отвечают | +| Restart всех stores и телеметрии | данные и сессия сохраняются | PASS | потерь нет | +| Нагрузка worker 6×40 Мп + 20 малых | не более 2 задач, память в лимите | все ready за 13.1 с, пик RSS 608 MiB / 768 | без OOM | + +Накладные расходы телеметрии не измерялись (трек 5 не заявлен). ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Collector, Prometheus, Loki, Tempo, Grafana, broker, PostgreSQL, S3 и HAProxy — единичные экземпляры. Остановка Collector приводит к потере телеметрии на время простоя (SDK держит не больше 2048 записей на сигнал и повторяет экспорт до 8 с). +- Collector не имеет дисковой очереди. Данные за время недоступности хранилища теряются, потери видны в `otelcol_exporter_send_failed_*`. +- Alerts проверены на коротких окнах (1 мин). Многооконный burn-rate (трек 5) не реализован, SLO за 30 суток на стенде не накапливаются. +- Метрики очереди берутся из PostgreSQL, а не из длины stream брокера: stream не обрезается (`XTRIM` не реализован), его длина не равна глубине очереди. +- `service.instance.id` — идентификатор контейнера: после пересоздания он меняется, старые ряды остаются в Prometheus до окончания retention. +- Трассы не сэмплируются (записывается всё). Цену наблюдаемости не измеряли. +- Порты Grafana, Prometheus, Loki, Tempo привязаны к 127.0.0.1; Prometheus, Loki и Tempo без аутентификации: допустимо только для локального стенда. +- Секреты лежат в `./secrets` вне Git. Права 644 нужны для чтения процессами контейнеров, это допустимо только для локального стенда. diff --git a/attachments.go b/attachments.go new file mode 100644 index 0000000..bd78cd0 --- /dev/null +++ b/attachments.go @@ -0,0 +1,534 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "hash" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "log/slog" + "net/http" + "os" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +const ( + objectPrefix = "attachments/" + stagingPrefix = "staging/" + probeHead = 256 << 10 + tailLen = 12 +) + +var ( + pngMagic = []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'} + pngTail = []byte{0, 0, 0, 0, 'I', 'E', 'N', 'D', 0xae, 0x42, 0x60, 0x82} + jpegMagic = []byte{0xff, 0xd8, 0xff} + jpegTail = []byte{0xff, 0xd9} +) + +type limits struct { + maxBytes int64 + maxPixels int64 + uploadSlots int + maxAttempts int +} + +type attachment struct { + ID, ChatID, OwnerID, Filename, ContentType, SHA256, Key, Status string + ErrorCode *string + Size int64 + Created, Updated time.Time + Variants []variant +} + +type variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + Size int64 `json:"size_bytes"` + Width int `json:"width"` + Height int `json:"height"` + SHA256 string `json:"sha256"` + key string +} + +func (x *attachment) json() map[string]any { + variants := x.Variants + if variants == nil { + variants = []variant{} + } + return map[string]any{ + "id": x.ID, + "chat_id": x.ChatID, + "owner_id": x.OwnerID, + "filename": x.Filename, + "content_type": x.ContentType, + "size_bytes": x.Size, + "sha256": x.SHA256, + "status": x.Status, + "variants": variants, + "error_code": x.ErrorCode, + "created_at": formatTime(x.Created), + "updated_at": formatTime(x.Updated), + } +} + +type probe struct { + sum hash.Hash + size int64 + head []byte + tail []byte +} + +func newProbe() *probe { + return &probe{sum: sha256.New()} +} + +func (p *probe) Write(b []byte) (int, error) { + p.sum.Write(b) + p.size += int64(len(b)) + if room := probeHead - len(p.head); room > 0 { + if room > len(b) { + room = len(b) + } + p.head = append(p.head, b[:room]...) + } + p.tail = append(p.tail, b...) + if len(p.tail) > tailLen { + p.tail = append(p.tail[:0], p.tail[len(p.tail)-tailLen:]...) + } + return len(b), nil +} + +func (p *probe) digest() string { + return hex.EncodeToString(p.sum.Sum(nil)) +} + +var ( + errNotImage = errors.New("not an image") + errTooBig = errors.New("too large") +) + +func (p *probe) verify(contentType string, maxPixels int64) error { + cfg, format, err := image.DecodeConfig(bytes.NewReader(p.head)) + if err != nil { + return errNotImage + } + var tail []byte + switch contentType { + case "image/png": + if format != "png" { + return errNotImage + } + tail = pngTail + case "image/jpeg": + if format != "jpeg" { + return errNotImage + } + tail = jpegTail + } + if !bytes.HasSuffix(p.tail, tail) { + return errNotImage + } + if int64(cfg.Width)*int64(cfg.Height) > maxPixels { + return errTooBig + } + return nil +} + +func magicMatches(contentType string, head []byte) bool { + switch contentType { + case "image/png": + return bytes.HasPrefix(head, pngMagic) + case "image/jpeg": + return bytes.HasPrefix(head, jpegMagic) + } + return false +} + +func validFilename(name string) bool { + n := utf8.RuneCountInString(name) + if n < 1 || n > 255 || !utf8.ValidString(name) { + return false + } + for _, r := range name { + if r == '/' || r == '\\' || r == 0 || unicode.IsControl(r) { + return false + } + } + return true +} + +type trackedReader struct { + r io.Reader + err error +} + +func (t *trackedReader) Read(b []byte) (int, error) { + n, err := t.r.Read(b) + if err != nil && err != io.EOF && t.err == nil { + t.err = err + } + return n, err +} + +func isTooLarge(err error) bool { + var mbe *http.MaxBytesError + return errors.As(err, &mbe) +} + +func (a *App) dropObject(key string) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := a.store.Remove(ctx, key); err != nil { + a.log.Warn("object cleanup failed", "key", key, "err", err.Error()) + } +} + +func (a *App) handleUploadAttachment(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + chatID := r.PathValue("chat_id") + if !isUUID(chatID) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id") + return + } + ok, err := a.isMember(r.Context(), chatID, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if !ok { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + select { + case a.uploads <- struct{}{}: + defer func() { <-a.uploads }() + default: + w.Header().Set("Retry-After", "1") + a.writeErr(w, reqID, 429, "rate_limited", "too many concurrent uploads") + return + } + filename := r.URL.Query().Get("filename") + if !validFilename(filename) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid filename") + return + } + contentType := strings.ToLower(strings.TrimSpace(strings.Split(r.Header.Get("Content-Type"), ";")[0])) + if contentType != "image/png" && contentType != "image/jpeg" { + a.writeErr(w, reqID, 415, "unsupported_media_type", "Content-Type must be image/png or image/jpeg") + return + } + if r.ContentLength > a.limits.maxBytes { + a.writeErr(w, reqID, 413, "payload_too_large", "file too large") + return + } + body := &trackedReader{r: http.MaxBytesReader(w, r.Body, a.limits.maxBytes)} + head := make([]byte, 16) + n, _ := io.ReadFull(body, head) + if body.err != nil { + if isTooLarge(body.err) { + a.writeErr(w, reqID, 413, "payload_too_large", "file too large") + } else { + a.writeErr(w, reqID, 400, "invalid_request", "upload interrupted") + } + return + } + head = head[:n] + if !magicMatches(contentType, head) { + a.writeErr(w, reqID, 415, "unsupported_media_type", "content does not match image type") + return + } + + id := newUUID() + trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("attachment.id", id)) + key := stagingPrefix + id + pr := newProbe() + size := r.ContentLength + if size < 0 { + size = -1 + } + src := io.TeeReader(io.MultiReader(bytes.NewReader(head), body), pr) + putCtx, cancel := context.WithTimeout(r.Context(), 60*time.Second) + defer cancel() + if err := a.store.Put(putCtx, key, src, size, contentType); err != nil { + a.dropObject(key) + switch { + case isTooLarge(body.err): + a.writeErr(w, reqID, 413, "payload_too_large", "file too large") + case body.err != nil: + a.writeErr(w, reqID, 400, "invalid_request", "upload interrupted") + default: + a.log.ErrorContext(r.Context(), "s3 put failed", "request_id", reqID, "err", err.Error()) + a.writeErr(w, reqID, 503, "unavailable", "storage unavailable") + } + return + } + if err := pr.verify(contentType, a.limits.maxPixels); err != nil { + a.dropObject(key) + if err == errTooBig { + a.writeErr(w, reqID, 413, "payload_too_large", "image too large") + } else { + a.writeErr(w, reqID, 415, "unsupported_media_type", "invalid image") + } + return + } + + now := nowUTC() + att := &attachment{ + ID: id, ChatID: chatID, OwnerID: u.ID, Filename: filename, ContentType: contentType, + SHA256: pr.digest(), Size: pr.size, Status: "queued", Created: now, Updated: now, + } + if err := a.enqueue(r.Context(), att, key); err != nil { + a.dropObject(key) + a.log.ErrorContext(r.Context(), "enqueue failed", "request_id", reqID, "err", err.Error()) + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if a.fault == "after_commit" { + a.log.Error("fault injected", "point", "after_commit", "attachment_id", att.ID) + os.Exit(3) + } + a.writeJSON(w, 202, att.json()) +} + +func (a *App) enqueue(ctx context.Context, att *attachment, key string) error { + ctx, cancel := dbCtx(ctx) + defer cancel() + tx, err := a.db.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + if _, err := tx.Exec(ctx, ` + insert into attachments(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key, status, created_at, updated_at) + values($1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, 'queued', $9, $9)`, + att.ID, att.ChatID, att.OwnerID, att.Filename, att.ContentType, att.Size, att.SHA256, key, att.Created); err != nil { + return err + } + jobID := newUUID() + tc := injectTrace(ctx) + if _, err := tx.Exec(ctx, ` + insert into jobs(id, attachment_id, pipeline_version, status, attempt, max_attempts, next_attempt_at, created_at, updated_at, traceparent, tracestate) + values($1::uuid, $2::uuid, $3, 'queued', 1, $4, $5, $5, $5, nullif($6, ''), nullif($7, ''))`, + jobID, att.ID, pipelineVersion, a.limits.maxAttempts, att.Created, tc.traceparent, tc.tracestate); err != nil { + return err + } + if _, err := tx.Exec(ctx, ` + insert into outbox(job_id, attempt, available_at, created_at) values($1::uuid, 1, $2, $2)`, jobID, att.Created); err != nil { + return err + } + return tx.Commit(ctx) +} + +func (a *App) loadAttachment(ctx context.Context, id, userID string) (*attachment, error) { + var x attachment + err := a.db.QueryRow(ctx, ` + select a.id::text, a.chat_id::text, a.owner_id::text, a.filename, a.content_type, + a.size_bytes, a.sha256, coalesce(a.object_key, ''), a.status, a.error_code, a.created_at, a.updated_at + from attachments a + join chat_members m on m.chat_id = a.chat_id and m.user_id = $2::uuid + where a.id = $1::uuid`, id, userID). + Scan(&x.ID, &x.ChatID, &x.OwnerID, &x.Filename, &x.ContentType, &x.Size, &x.SHA256, &x.Key, &x.Status, &x.ErrorCode, &x.Created, &x.Updated) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if x.Status != "ready" { + return &x, nil + } + rows, err := a.db.Query(ctx, ` + select name, content_type, size_bytes, width, height, sha256, object_key + from attachment_variants where attachment_id = $1::uuid and pipeline_version = $2 + order by array_position(array['thumbnail', 'crop', 'optimized'], name)`, id, pipelineVersion) + if err != nil { + return nil, err + } + x.Variants, err = pgx.CollectRows(rows, func(row pgx.CollectableRow) (variant, error) { + var v variant + return v, row.Scan(&v.Name, &v.ContentType, &v.Size, &v.Width, &v.Height, &v.SHA256, &v.key) + }) + if err != nil { + return nil, err + } + return &x, nil +} + +func (a *App) attachmentFor(w http.ResponseWriter, r *http.Request) *attachment { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return nil + } + id := r.PathValue("attachment_id") + if !isUUID(id) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid attachment_id") + return nil + } + ctx, cancel := dbCtx(r.Context()) + defer cancel() + att, err := a.loadAttachment(ctx, id, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return nil + } + if att == nil { + a.writeErr(w, reqID, 404, "not_found", "attachment not found") + return nil + } + return att +} + +func (a *App) handleGetAttachment(w http.ResponseWriter, r *http.Request) { + if att := a.attachmentFor(w, r); att != nil { + a.writeJSON(w, 200, att.json()) + } +} + +func (a *App) handleDownloadAttachment(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + name := r.URL.Query().Get("variant") + if name == "" { + name = "original" + } + if name != "original" && name != "thumbnail" && name != "crop" && name != "optimized" { + a.writeErr(w, reqID, 400, "invalid_request", "invalid variant") + return + } + att := a.attachmentFor(w, r) + if att == nil { + return + } + if att.Status != "ready" { + a.writeErr(w, reqID, 409, "conflict", "attachment is not ready") + return + } + key, contentType := att.Key, att.ContentType + if name != "original" { + key = "" + for _, v := range att.Variants { + if v.Name == name { + key, contentType = v.key, v.ContentType + } + } + if key == "" { + a.writeErr(w, reqID, 404, "not_found", "variant not found") + return + } + } + ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) + defer cancel() + obj, info, err := a.store.Get(ctx, key) + if err != nil { + if isNoSuchKey(err) { + a.writeErr(w, reqID, 404, "not_found", "attachment not found") + return + } + a.log.ErrorContext(r.Context(), "s3 get failed", "request_id", reqID, "err", err.Error()) + a.writeErr(w, reqID, 503, "unavailable", "storage unavailable") + return + } + defer obj.Close() + first := make([]byte, 32<<10) + n, err := io.ReadFull(obj, first) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + a.log.ErrorContext(r.Context(), "s3 read failed", "request_id", reqID, "err", err.Error()) + a.writeErr(w, reqID, 503, "unavailable", "storage unavailable") + return + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", strconv.FormatInt(info.Size, 10)) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(200) + _, _ = w.Write(first[:n]) + _, _ = io.Copy(w, obj) +} + +var keepQueries = map[string]string{ + objectPrefix: ` + select k from unnest($1::text[]) k + where exists(select 1 from attachments where object_key = k) + or exists(select 1 from attachment_variants where object_key = k)`, + stagingPrefix: ` + select k from unnest($1::text[]) k + where exists(select 1 from attachments where staging_key = k and (status in ('queued', 'processing') + or (status = 'failed' and updated_at > now() - make_interval(secs => $2))))`, +} + +func cleanupOrphans(ctx context.Context, db *pgxpool.Pool, store *Store, grace, retention time.Duration, log *slog.Logger) (int, error) { + cutoff := time.Now().Add(-grace) + removed := 0 + flush := func(prefix string, keys []string) error { + if len(keys) == 0 { + return nil + } + args := []any{keys} + if prefix == stagingPrefix { + args = append(args, retention.Seconds()) + } + rows, err := db.Query(ctx, keepQueries[prefix], args...) + if err != nil { + return err + } + kept, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return err + } + known := map[string]bool{} + for _, k := range kept { + known[k] = true + } + for _, k := range keys { + if known[k] { + continue + } + if err := store.Remove(ctx, k); err != nil { + return err + } + log.Info("orphan object removed", "key", k) + removed++ + } + return nil + } + for _, prefix := range []string{objectPrefix, stagingPrefix} { + var batch []string + for obj := range store.List(ctx, prefix) { + if obj.Err != nil { + return removed, obj.Err + } + if obj.LastModified.After(cutoff) { + continue + } + batch = append(batch, obj.Key) + if len(batch) == 500 { + if err := flush(prefix, batch); err != nil { + return removed, err + } + batch = batch[:0] + } + } + if err := flush(prefix, batch); err != nil { + return removed, err + } + } + return removed, nil +} diff --git a/auth.go b/auth.go new file mode 100644 index 0000000..cacc42d --- /dev/null +++ b/auth.go @@ -0,0 +1,131 @@ +package main + +import ( + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" + "unicode/utf8" + + "golang.org/x/crypto/argon2" +) + +const ( + argonTime = 2 + argonMemory = 19456 + argonThreads = 1 + argonKeyLen = 32 + argonSaltLen = 16 +) + +func hashPassword(password string) (string, error) { + salt := make([]byte, argonSaltLen) + if _, err := rand.Read(salt); err != nil { + return "", err + } + hash := argon2.IDKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen) + b64Salt := base64.RawStdEncoding.EncodeToString(salt) + b64Hash := base64.RawStdEncoding.EncodeToString(hash) + return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", + argon2.Version, argonMemory, argonTime, argonThreads, b64Salt, b64Hash), nil +} + +func checkPassword(encoded, password string) (bool, error) { + parts := strings.Split(encoded, "$") + if len(parts) != 6 || parts[1] != "argon2id" { + return false, errors.New("bad hash") + } + var version int + if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil { + return false, err + } + var memory, timeCost, threads int + for _, p := range strings.Split(parts[3], ",") { + kv := strings.SplitN(p, "=", 2) + if len(kv) != 2 { + return false, errors.New("bad params") + } + n, err := strconv.Atoi(kv[1]) + if err != nil { + return false, err + } + switch kv[0] { + case "m": + memory = n + case "t": + timeCost = n + case "p": + threads = n + } + } + salt, err := base64.RawStdEncoding.DecodeString(parts[4]) + if err != nil { + return false, err + } + want, err := base64.RawStdEncoding.DecodeString(parts[5]) + if err != nil { + return false, err + } + got := argon2.IDKey([]byte(password), salt, uint32(timeCost), uint32(memory), uint8(threads), uint32(len(want))) + return subtle.ConstantTimeCompare(got, want) == 1, nil +} + +func newToken() (string, []byte, string, error) { + raw := make([]byte, 32) + if _, err := rand.Read(raw); err != nil { + return "", nil, "", err + } + token := base64.RawURLEncoding.EncodeToString(raw) + sum := sha256.Sum256([]byte(token)) + hash := make([]byte, len(sum)) + copy(hash, sum[:]) + return token, hash, hex.EncodeToString(hash), nil +} + +func tokenHash(token string) ([]byte, string) { + sum := sha256.Sum256([]byte(token)) + hash := make([]byte, len(sum)) + copy(hash, sum[:]) + return hash, hex.EncodeToString(hash) +} + +func parseBasic(header string) (username, password string, ok bool) { + const prefix = "Basic " + if !strings.HasPrefix(header, prefix) { + return "", "", false + } + decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(header[len(prefix):])) + if err != nil { + return "", "", false + } + s := string(decoded) + i := strings.IndexByte(s, ':') + if i < 0 { + return "", "", false + } + username = s[:i] + password = s[i+1:] + return username, password, true +} + +func parseBearer(header string) (string, bool) { + const prefix = "Bearer " + if !strings.HasPrefix(header, prefix) { + return "", false + } + tok := strings.TrimSpace(header[len(prefix):]) + if tok == "" { + return "", false + } + return tok, true +} + +func validPassword(p string) bool { + n := utf8.RuneCountInString(p) + return n >= 12 && n <= 128 +} diff --git a/broker.go b/broker.go new file mode 100644 index 0000000..fea8e4b --- /dev/null +++ b/broker.go @@ -0,0 +1,162 @@ +package main + +import ( + "context" + "encoding/json" + "log/slog" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +const ( + jobStream = "attachments.jobs" + jobGroup = "workers" + jobType = "attachment.process.v1" + pipelineVersion = 1 +) + +type jobMessage struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt string `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +type Broker struct { + rdb *redis.Client +} + +func openBroker() (*Broker, error) { + raw, err := readSecret("BROKER_URL", "BROKER_URL_FILE") + if err != nil { + return nil, err + } + opt, err := redis.ParseURL(raw) + if err != nil { + return nil, err + } + opt.DialTimeout = 2 * time.Second + opt.ReadTimeout = 3 * time.Second + opt.WriteTimeout = 3 * time.Second + opt.MaxRetries = 0 + return &Broker{rdb: redis.NewClient(opt)}, nil +} + +func (b *Broker) ensureGroup(ctx context.Context) error { + err := b.rdb.XGroupCreateMkStream(ctx, jobStream, jobGroup, "0").Err() + if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") { + return err + } + return nil +} + +func (b *Broker) publish(ctx context.Context, m jobMessage) error { + body, err := json.Marshal(m) + if err != nil { + return err + } + return b.rdb.XAdd(ctx, &redis.XAddArgs{Stream: jobStream, Values: map[string]any{"job": string(body)}}).Err() +} + +func (b *Broker) publishTraced(ctx context.Context, m jobMessage) (err error) { + if m.Traceparent == "" { + return b.publish(ctx, m) + } + ctx = extractTrace(ctx, traceCarrier{m.Traceparent, m.Tracestate}) + ctx, sp := tracer.Start(ctx, jobStream+" publish", trace.WithSpanKind(trace.SpanKindProducer), trace.WithAttributes( + attribute.String("messaging.system", "redis"), + attribute.String("messaging.destination.name", jobStream), + attribute.String("messaging.operation.type", "publish"), + attribute.String("job.id", m.JobID), + attribute.String("attachment.id", m.AttachmentID), + attribute.Int("job.attempt", m.Attempt), + )) + defer func() { endSpan(sp, err) }() + c := injectTrace(ctx) + m.Traceparent, m.Tracestate = c.traceparent, "" + if len(c.tracestate) <= 512 { + m.Tracestate = c.tracestate + } + return b.publish(ctx, m) +} + +func relayOnce(ctx context.Context, db *pgxpool.Pool, b *Broker) (int, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + tx, err := db.Begin(ctx) + if err != nil { + return 0, err + } + defer tx.Rollback(ctx) + rows, err := tx.Query(ctx, ` + select o.id, o.job_id::text, o.attempt, j.attachment_id::text, j.pipeline_version, j.created_at, + coalesce(j.traceparent, ''), coalesce(j.tracestate, '') + from outbox o + join jobs j on j.id = o.job_id + where o.published_at is null and o.available_at <= now() + order by o.id + limit 50 + for update of o skip locked`) + if err != nil { + return 0, err + } + var ids []int64 + var msgs []jobMessage + for rows.Next() { + var id int64 + var created time.Time + m := jobMessage{SchemaVersion: 1, Type: jobType} + if err := rows.Scan(&id, &m.JobID, &m.Attempt, &m.AttachmentID, &m.PipelineVersion, &created, &m.Traceparent, &m.Tracestate); err != nil { + rows.Close() + return 0, err + } + m.CreatedAt = formatTime(created) + ids = append(ids, id) + msgs = append(msgs, m) + } + rows.Close() + if err := rows.Err(); err != nil { + return 0, err + } + if len(ids) == 0 { + return 0, nil + } + if err := b.ensureGroup(ctx); err != nil { + return 0, err + } + for _, m := range msgs { + if err := b.publishTraced(ctx, m); err != nil { + return 0, err + } + } + if _, err := tx.Exec(ctx, `update outbox set published_at = now() where id = any($1)`, ids); err != nil { + return 0, err + } + return len(ids), tx.Commit(ctx) +} + +func runRelay(ctx context.Context, db *pgxpool.Pool, b *Broker, log *slog.Logger) { + for ctx.Err() == nil { + n, err := relayOnce(ctx, db, b) + if err != nil { + log.Warn("outbox relay failed", "err", err.Error()) + time.Sleep(2 * time.Second) + continue + } + if n > 0 { + log.Info("outbox published", "count", n) + continue + } + time.Sleep(300 * time.Millisecond) + } +} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..2f4a5d3 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,475 @@ +x-api-env: &api-env + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + SESSION_STORE: postgres + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-10} + GOMEMLIMIT: 150MiB + SHUTDOWN_TIMEOUT_SECONDS: 10 + DRAIN_SECONDS: 3 + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + BROKER_URL_FILE: /run/secrets/broker_url + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + FAULT_POINT: ${API_FAULT_POINT:-} + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${APP_ENV:-lab} + +services: + postgres: + image: postgres:17-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24 + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_DB: messenger + secrets: + - postgres_password + volumes: + - pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + networks: + - backend + restart: unless-stopped + + valkey: + image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: ["valkey-server", "--appendonly", "yes", "--save", "60", "1"] + volumes: + - valkey_data:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: unless-stopped + + broker: + image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: ["valkey-server", "/run/secrets/broker_conf", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory-policy", "noeviction"] + secrets: + - broker_conf + volumes: + - broker_data:/data + healthcheck: + test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$(sed -n 's/^requirepass //p' /run/secrets/broker_conf)\" ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: unless-stopped + + s3: + image: chrislusf/seaweedfs@sha256:c2658820e763294a9e9de7b36c81e59b879e3459148351aee4e31a126ca64860 + command: + - server + - -dir=/data + - -ip=s3 + - -master.volumeSizeLimitMB=128 + - -volume.max=30 + - -s3 + - -s3.port=9000 + - -s3.config=/run/secrets/s3_config + secrets: + - s3_config + volumes: + - s3_data:/data + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9333/dir/assign | grep -q fid"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + networks: + - storage + restart: unless-stopped + + s3-init: + image: mtusi-messenger:lab6 + command: ["s3-init"] + environment: + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + depends_on: + s3: + condition: service_healthy + migrate: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - storage + restart: "no" + + janitor: + image: mtusi-messenger:lab6 + command: ["janitor"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + CLEANUP_GRACE_SECONDS: ${CLEANUP_GRACE_SECONDS:-600} + CLEANUP_INTERVAL_SECONDS: ${CLEANUP_INTERVAL_SECONDS:-300} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3-init: + condition: service_completed_successfully + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.25" + memory: 64M + networks: + - backend + - storage + restart: unless-stopped + + worker: + image: mtusi-messenger:lab6 + command: ["worker"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_BACKOFF_BASE_MS: ${JOB_BACKOFF_BASE_MS:-1000} + JOB_BACKOFF_MAX_MS: ${JOB_BACKOFF_MAX_MS:-30000} + GOMEMLIMIT: 600MiB + FAULT_POINT: ${WORKER_FAULT_POINT:-} + OTEL_SERVICE_NAME: messenger-worker + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${APP_ENV:-lab} + secrets: + - database_url + - broker_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3-init: + condition: service_completed_successfully + broker: + condition: service_healthy + otel-collector: + condition: service_started + healthcheck: + disable: true + stop_grace_period: 30s + deploy: + resources: + limits: + cpus: "1.0" + memory: 768M + networks: + - backend + - storage + restart: unless-stopped + + migrate: + build: . + image: mtusi-messenger:lab6 + command: ["migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: "no" + + api: + image: mtusi-messenger:lab6 + environment: *api-env + secrets: + - database_url + - cache_url + - broker_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3-init: + condition: service_completed_successfully + valkey: + condition: service_healthy + broker: + condition: service_healthy + otel-collector: + condition: service_started + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "0.5" + memory: 192M + networks: + - backend + - storage + restart: unless-stopped + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.116.1@sha256:d0ebf65280da2e1b1491d1b93648281afd353d4b9ea19160090303cec9a233bd + command: ["--config=/etc/otelcol/config.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro + depends_on: + prometheus: + condition: service_healthy + loki: + condition: service_healthy + tempo: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + networks: + - backend + - telemetry + restart: unless-stopped + + prometheus: + image: prom/prometheus:v3.1.0@sha256:6559acbd5d770b15bb3c954629ce190ac3cbbdb2b7f1c30f0385c4e05104e218 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + - --web.enable-otlp-receiver + ports: + - "127.0.0.1:9090:9090" + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/rules:/etc/prometheus/rules:ro + - prometheus_data:/prometheus + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/ready"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.5" + memory: 384M + networks: + - telemetry + restart: unless-stopped + + loki: + image: grafana/loki:3.3.2@sha256:8af2de1abbdd7aa92b27c9bcc96f0f4140c9096b507c77921ffddf1c6ad6c48f + command: ["-config.file=/etc/loki/loki.yaml"] + ports: + - "127.0.0.1:3100:3100" + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - loki_data:/loki + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3100/ready"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 15s + deploy: + resources: + limits: + cpus: "0.5" + memory: 384M + networks: + - telemetry + restart: unless-stopped + + tempo: + image: grafana/tempo:2.7.0@sha256:12e904cc509a8e0bd87815e0bc719b6167933bc3f2532716a85811af967780f6 + command: ["-config.file=/etc/tempo/tempo.yaml"] + ports: + - "127.0.0.1:3200:3200" + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo_data:/var/tempo + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3200/ready"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 15s + deploy: + resources: + limits: + cpus: "0.5" + memory: 384M + networks: + - telemetry + restart: unless-stopped + + grafana: + image: grafana/grafana:11.3.0@sha256:a0f881232a6fb71a0554a47d0fe2203b6888fe77f4cefb7ea62bed7eb54e13c3 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password + GF_USERS_ALLOW_SIGN_UP: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + secrets: + - grafana_admin_password + ports: + - "127.0.0.1:3000:3000" + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + depends_on: + prometheus: + condition: service_healthy + loki: + condition: service_healthy + tempo: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 15s + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + networks: + - telemetry + restart: unless-stopped + + lb: + image: haproxy:3.0-alpine@sha256:cf3af0facf7eb22b9fee463daf0fbc07ead740b8150696dce724264faf7b4318 + ports: + - "127.0.0.1:8080:8080" + volumes: + - ./lb/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + - api + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: unless-stopped + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + broker_conf: + file: ./secrets/broker.conf + broker_url: + file: ./secrets/broker_url + s3_config: + file: ./secrets/s3_config.json + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + grafana_admin_password: + file: ./secrets/grafana_admin_password + +volumes: + pg_data: + valkey_data: + broker_data: + s3_data: + prometheus_data: + loki_data: + tempo_data: + grafana_data: + +networks: + backend: + driver: bridge + storage: + driver: bridge + telemetry: + driver: bridge diff --git a/db.go b/db.go new file mode 100644 index 0000000..371c9bf --- /dev/null +++ b/db.go @@ -0,0 +1,59 @@ +package main + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func readSecret(envKey, fileKey string) (string, error) { + if v := os.Getenv(envKey); v != "" { + return strings.TrimSpace(v), nil + } + path := os.Getenv(fileKey) + if path == "" { + return "", fmt.Errorf("%s or %s required", envKey, fileKey) + } + b, err := os.ReadFile(path) + if err != nil { + return "", err + } + return strings.TrimSpace(string(b)), nil +} + +func openDB(ctx context.Context) (*pgxpool.Pool, error) { + dsn, err := readSecret("DATABASE_URL", "DATABASE_URL_FILE") + if err != nil { + return nil, err + } + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + return nil, err + } + cfg.MaxConns = 10 + cfg.MinConns = 2 + cfg.MaxConnLifetime = 5 * time.Minute + cfg.ConnConfig.ConnectTimeout = 3 * time.Second + cfg.ConnConfig.Tracer = dbTracer{} + + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + return nil, err + } + pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := pool.Ping(pingCtx); err != nil { + pool.Close() + return nil, err + } + registerPoolMetrics(pool) + return pool, nil +} + +func dbCtx(parent context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(parent, 3*time.Second) +} diff --git a/evidence/attachments.txt b/evidence/attachments.txt new file mode 100644 index 0000000..6e382b8 --- /dev/null +++ b/evidence/attachments.txt @@ -0,0 +1 @@ +PASS: attachment validation, limits and access control diff --git a/evidence/dashboard.txt b/evidence/dashboard.txt new file mode 100644 index 0000000..b037e6b --- /dev/null +++ b/evidence/dashboard.txt @@ -0,0 +1,2 @@ +grafana: 3 datasources and dashboard "Messenger observability" provisioned from Git, 28 panels, 34 queries executed without errors +PASS: provisioning and dashboard queries diff --git a/evidence/failover-1.txt b/evidence/failover-1.txt new file mode 100644 index 0000000..96cc3b2 --- /dev/null +++ b/evidence/failover-1.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 138, "failed_attempts": 0, "first_recovery_seconds": 3.483} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-2.txt b/evidence/failover-2.txt new file mode 100644 index 0000000..1967b2c --- /dev/null +++ b/evidence/failover-2.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 138, "failed_attempts": 0, "first_recovery_seconds": 3.506} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-each.json b/evidence/failover-each.json new file mode 100644 index 0000000..ab270eb --- /dev/null +++ b/evidence/failover-each.json @@ -0,0 +1,71 @@ +{ + "baseline": { + "window_seconds": 6, + "successful_requests": 2689, + "failed_attempts": 0, + "successful_rps": 448.2, + "last_error_offset_seconds": null, + "instances": [ + "1dbf9e0d259e", + "638505d2d88b" + ] + }, + "replicas": [ + { + "stopped_replica": "638505d2d88b", + "during_failure": { + "window_seconds": 15.37, + "successful_requests": 4298, + "failed_attempts": 7, + "successful_rps": 279.7, + "last_error_offset_seconds": 1.06, + "instances": [ + "1dbf9e0d259e", + "638505d2d88b" + ] + }, + "seen_stopped_replica_after_stop": false, + "stop_command_seconds": 3.37, + "restore_seconds": 6.79, + "after_restore": { + "window_seconds": 7.0, + "successful_requests": 3123, + "failed_attempts": 0, + "successful_rps": 446.1, + "last_error_offset_seconds": null, + "instances": [ + "1dbf9e0d259e", + "638505d2d88b" + ] + } + }, + { + "stopped_replica": "1dbf9e0d259e", + "during_failure": { + "window_seconds": 15.49, + "successful_requests": 4470, + "failed_attempts": 2, + "successful_rps": 288.5, + "last_error_offset_seconds": 1.04, + "instances": [ + "1dbf9e0d259e", + "638505d2d88b" + ] + }, + "seen_stopped_replica_after_stop": false, + "stop_command_seconds": 3.49, + "restore_seconds": 6.8, + "after_restore": { + "window_seconds": 7.0, + "successful_requests": 3129, + "failed_attempts": 0, + "successful_rps": 447.0, + "last_error_offset_seconds": null, + "instances": [ + "1dbf9e0d259e", + "638505d2d88b" + ] + } + } + ] +} diff --git a/evidence/faults.txt b/evidence/faults.txt new file mode 100644 index 0000000..e65dbdb --- /dev/null +++ b/evidence/faults.txt @@ -0,0 +1,7 @@ +1) api crashed after COMMIT before 202/publish: attachment=8bcd0ac3-c34d-45d0-8f48-55a0283f7832 recovered by outbox relay (published rows=1) -> ready attempt=1 +2) worker crashed after S3 write before COMMIT: lease expired, reaper requeued, attempt=2 ready, variants=3 +3) worker crashed after COMMIT before ack: message redelivered after restart, skipped as no-op, attempt=1, variants=3 +4) s3 down: transient errors retried with backoff+jitter delays_ms=[895, 1219, 3028, 5245], then failed retry_exhausted, listed in registry + replay: failed -> queued -> ready (audit rows=1); replay of ready job skipped, variants=3 +5) corrupted IDAT (passes header/tail check in API): failed invalid_image on attempt 1, no retries +PASS: outbox recovery, lease recovery, redelivery after commit, backoff with jitter, retry_exhausted, replay, permanent error diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt new file mode 100644 index 0000000..50987a9 --- /dev/null +++ b/evidence/idempotency.txt @@ -0,0 +1 @@ +PASS: 32 parallel requests, 1 message, 2 instances diff --git a/evidence/incident-database.txt b/evidence/incident-database.txt new file mode 100644 index 0000000..5a7943b --- /dev/null +++ b/evidence/incident-database.txt @@ -0,0 +1,4 @@ +2) database lock on messages: ApiHighErrorRate firing after 52s; 5xx ratio=0.47 only on ['/api/v1/chats/{chat_id}/messages']; p99=4.9s; pool used=0.0 acquire wait=0.01 s/s + diagnosis from stored data: error trace 17a3258b1897cd36f88418fb1adc23c5, server span 3006 ms, child "db INSERT" 2997 ms (100%) with status error, handler has no other slow child; requests: {503: 52, 201: 164} + restored: lock released, alert resolved after 40s +PASS: slow database detected by alert and diagnosed from telemetry diff --git a/evidence/incident-storage.txt b/evidence/incident-storage.txt new file mode 100644 index 0000000..406fb1d --- /dev/null +++ b/evidence/incident-storage.txt @@ -0,0 +1,5 @@ +3) prometheus, loki and tempo stopped for ~80s under load: requests={201: 942, 'upload': 99}, 3/3 uploads ready, message 201, no 5xx + collector rss MiB: start=64 peak=71 during outage (limit 256, memory_limiter 160), restarts/oom=0 false + telemetry lost and counted: collector send_failed spans +2443, log records +254, metric points +456; sdk export errors=0 + after restore new logs and metrics are stored again; data from the outage window is lost, data before it remains +PASS: telemetry storage outage does not affect the API and loss is bounded and counted diff --git a/evidence/incident-worker.txt b/evidence/incident-worker.txt new file mode 100644 index 0000000..55bc029 --- /dev/null +++ b/evidence/incident-worker.txt @@ -0,0 +1,4 @@ +1) worker stopped: QueueLagGrowing firing after 60s; oldest job age=58s depth=57 jobs/s=0.00 worker_busy=0.0 + diagnosis from stored data: trace 4710543929ec0f33fc13ce8565b5bc6a has the api server and "attachments.jobs publish" spans and no consumer span; no "job ready" worker log in Loki during the last minute (0 lines) + restored: worker started, alert resolved after 20s, backlog drained in 21s +PASS: stopped worker detected by alert and diagnosed from telemetry diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 0000000..97ecea3 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.116, + "concurrency": 4, + "successful_requests": 11666, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 387.368, + "successful_latency_ms": { + "p50": 5.802, + "p95": 44.802, + "p99": 55.465 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/observability.txt b/evidence/observability.txt new file mode 100644 index 0000000..15947bd --- /dev/null +++ b/evidence/observability.txt @@ -0,0 +1,5 @@ +chain: request_id=77a5a56d-55b7-4fb1-9849-a4c39d03d114 trace_id=391ac5aeb8757b980131897c9b96322b spans=30 api->producer->consumer->s3/db linked, api and worker logs carry trace_id +metrics: 13 signals present, labels=['deployment_environment_name', 'http_request_method', 'http_response_status_class', 'http_route', 'instance', 'job', 'job_type', 'le', 'outcome', 'queue', 'service_version', 'state'], routes are templates (10) +secrets: message body and session token absent from Loki and Tempo +retry: one trace 9bb4fe6dc99b3dbde778e1758a85acd9 holds 6 attempts (5 failed with S3 down, attempt 6 ready after replay) +PASS: signals, producer/consumer trace chain, log correlation, retry in trace, no secrets in telemetry diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..74e1ab5 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,32 @@ +== before +postgres 2026-11-19T19:48:24.393646212Z +valkey 2026-11-19T19:48:24.279274086Z +broker 2026-11-19T19:48:24.52160412Z +s3 2026-11-19T19:48:34.104469144Z +prometheus 2026-11-19T19:48:24.131549676Z +loki 2026-11-19T19:48:25.768218547Z +tempo 2026-11-19T19:48:25.585571452Z +otel-collector 2026-11-19T19:49:10.223513181Z +grafana 2026-11-19T19:49:10.079005478Z +api 2026-11-19T19:49:13.028782328Z +api 2026-11-19T19:49:13.082849306Z +lb 2026-11-19T19:49:10.26422772Z +janitor 2026-11-19T19:49:10.239020362Z +worker 2026-11-19T19:49:11.105495834Z +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +== after +postgres 2026-11-19T19:49:23.902795389Z +valkey 2026-11-19T19:49:23.736355641Z +broker 2026-11-19T19:49:23.770551967Z +s3 2026-11-19T19:49:33.778194628Z +prometheus 2026-11-19T19:49:23.880621282Z +loki 2026-11-19T19:49:25.646714025Z +tempo 2026-11-19T19:49:24.313008245Z +otel-collector 2026-11-19T19:50:10.305030304Z +grafana 2026-11-19T19:50:10.322614526Z +api 2026-11-19T19:50:13.127529668Z +api 2026-11-19T19:50:13.147392359Z +lb 2026-11-19T19:50:10.154157227Z +janitor 2026-11-19T19:50:10.272224264Z +worker 2026-11-19T19:50:14.68861531Z diff --git a/evidence/queue-recovery.txt b/evidence/queue-recovery.txt new file mode 100644 index 0000000..5cf545f --- /dev/null +++ b/evidence/queue-recovery.txt @@ -0,0 +1,5 @@ +accepted: attachment=e91cf4d7-c48d-43cd-9a1e-633ca1dabb90 job=ca2f9db6-c303-4663-b204-eb589ee11616 status=queued, job in stream, worker stopped +restarted broker and api: job still in stream, attachment still queued +worker started: attachment ready, attempt=1, variants=3 +duplicate delivery: skipped as no-op, acked, variants=3, updated_at unchanged +PASS: worker stop, broker/api restart, redelivery idempotency diff --git a/evidence/rolling.json b/evidence/rolling.json new file mode 100644 index 0000000..3137293 --- /dev/null +++ b/evidence/rolling.json @@ -0,0 +1,24 @@ +{ + "old_replicas": [ + "638505d2d88b", + "1dbf9e0d259e" + ], + "new_replicas": [ + "02a35d2fcbe0", + "20278f44f3e4" + ], + "update_window": { + "window_seconds": 16.31, + "successful_requests": 7541, + "failed_attempts": 2, + "successful_rps": 462.3, + "last_error_offset_seconds": 12.63, + "instances": [ + "02a35d2fcbe0", + "1dbf9e0d259e", + "20278f44f3e4", + "638505d2d88b" + ] + }, + "old_replicas_served_after_update": [] +} diff --git a/evidence/s3-checks.txt b/evidence/s3-checks.txt new file mode 100644 index 0000000..6964887 --- /dev/null +++ b/evidence/s3-checks.txt @@ -0,0 +1 @@ +PASS: private bucket, limited credentials, s3 outage, orphan and staging cleanup diff --git a/evidence/session-revocation.txt b/evidence/session-revocation.txt new file mode 100644 index 0000000..38dab27 --- /dev/null +++ b/evidence/session-revocation.txt @@ -0,0 +1 @@ +PASS: token revoked while valkey was stopped stays revoked after valkey restart (401) diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..566c729 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,4 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/telemetry-persistence.txt b/evidence/telemetry-persistence.txt new file mode 100644 index 0000000..4b8ef5a --- /dev/null +++ b/evidence/telemetry-persistence.txt @@ -0,0 +1,2 @@ +persistence: trace c23eec016505cb2f6cd5dd7d6b400c15 (30 spans), its request log and 18 metric series survived a restart of prometheus, loki and tempo +PASS: telemetry history is kept on volumes diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 0000000..e9ff765 --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,18 @@ +{ + "file_bytes": 9722608, + "concurrent_uploads": 12, + "rounds": 3, + "uploads_ok": 36, + "uploads_failed": 0, + "elapsed_seconds": 18.72, + "api_memory_limit_mib": 192, + "idle_memory_mib": { + "mtusi2026lab6-fork-api-1": 67.2, + "mtusi2026lab6-fork-api-2": 70.4 + }, + "peak_memory_mib": { + "mtusi2026lab6-fork-api-1": 115.8, + "mtusi2026lab6-fork-api-2": 135.4 + }, + "buffering_all_files_would_need_mib": 111.3 +} diff --git a/evidence/variants.txt b/evidence/variants.txt new file mode 100644 index 0000000..9a5e8bf --- /dev/null +++ b/evidence/variants.txt @@ -0,0 +1 @@ +PASS: thumbnail/crop/optimized sizes, formats, checksums, no upscale, alpha on white, EXIF orientation, no EXIF in variants diff --git a/evidence/worker-load.json b/evidence/worker-load.json new file mode 100644 index 0000000..0415a8b --- /dev/null +++ b/evidence/worker-load.json @@ -0,0 +1,26 @@ +{ + "huge_images": { + "count": 6, + "pixels": 39942400, + "bytes": 116545 + }, + "small_images": 20, + "worker_concurrency": 2, + "max_processing_jobs_observed": 2, + "all_ready": true, + "accept_seconds": 0.35, + "all_ready_seconds": 13.1, + "worker_peak_rss_mib": 607.7, + "worker_memory_limit_mib": 768, + "worker_restarts_oom": [ + "0", + "false" + ], + "timeout_case": { + "job_timeout_seconds": 1, + "status": "failed", + "error_code": "retry_exhausted", + "attempts": 5, + "last_error": "context deadline exceeded" + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d4b4159 --- /dev/null +++ b/go.mod @@ -0,0 +1,61 @@ +module mtusi-messenger + +go 1.26.0 + +require ( + github.com/gen2brain/webp v0.6.4 + github.com/jackc/pgx/v5 v5.11.0 + github.com/minio/minio-go/v7 v7.4.1 + github.com/redis/go-redis/v9 v9.23.0 + github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd + go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 + go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 + go.opentelemetry.io/otel/metric v1.47.0 + go.opentelemetry.io/otel/sdk v1.47.0 + go.opentelemetry.io/otel/sdk/log v1.47.0 + go.opentelemetry.io/otel/sdk/metric v1.47.0 + go.opentelemetry.io/otel/trace v1.47.0 + golang.org/x/crypto v0.58.0 + golang.org/x/image v0.47.0 +) + +require ( + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.10.1 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect + go.uber.org/atomic v1.12.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/net v0.60.0 // indirect + golang.org/x/sync v0.24.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/text v0.43.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/grpc v1.83.2 // indirect + google.golang.org/protobuf v1.36.12 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..fa49710 --- /dev/null +++ b/go.sum @@ -0,0 +1,142 @@ +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/gen2brain/webp v0.6.4 h1:SUDdmxADOAiPQ+5ylNmuHhuYf2dOi0KgKZHL5vpVCNU= +github.com/gen2brain/webp v0.6.4/go.mod h1:iGWMaCSw7t3I/Cv9llzEKmpnR36S8lS8VL/ZVjxU0JE= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w= +github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= +github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 h1:/CdgE52i1OEPHhsGUqrggYNRaZy4v4YgVenGd52dRBI= +go.opentelemetry.io/contrib/bridges/otelslog v0.21.0/go.mod h1:9pBdbVbuehIDzxN4accnzzxEUPT8Nq7grbZMo+J1VN4= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0 h1:5WVp40k2fJHuc5BtNTgKk+EugKFT8Eg3lywFfCPI/nE= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0/go.mod h1:59nH0RYCGHRlwt15WyioeSYINBbuObeBpTowPhEKWHY= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 h1:7kszRN4Ze5GeAkUnBjBSUIunVBMA+mZIG04auueqguM= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0/go.mod h1:CLzakYy8jOdkBfcveCWYkeKJxujcF4xPhh2Z7IfE98U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM= +go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/log v1.47.0 h1:W4N/ZfgpkRRnmARnqYoQ6BrWA0mrj8IzVDNLlpmUJZY= +go.opentelemetry.io/otel/sdk/log v1.47.0/go.mod h1:Wu74Z2EmOKXi2FleSNAb0Hyau2G+r5kubSEMJw5aAMs= +go.opentelemetry.io/otel/sdk/log/logtest v0.23.0 h1:j3pm42QCrKYYLLkI6OrV1QHFkUfDA6/4EP+Y1PzggLg= +go.opentelemetry.io/otel/sdk/log/logtest v0.23.0/go.mod h1:zt0gEnuTZgDvqvczHwYswz93eqMaIr6upTTsdV/HUlU= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= +go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo= +golang.org/x/image v0.47.0/go.mod h1:IkN5o4A7vq55vXN8FWjjnhg9Rs2GOtffqsSi9mWAKYE= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/handlers.go b/handlers.go new file mode 100644 index 0000000..2af3b48 --- /dev/null +++ b/handlers.go @@ -0,0 +1,1095 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "regexp" + "strconv" + "strings" + "sync/atomic" + "time" + "unicode/utf8" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" +) + +var idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +var instanceID = func() string { + h, err := os.Hostname() + if err != nil || h == "" { + return newUUID() + } + return h +}() + +var usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +type apiError struct { + Code string `json:"code"` + Message string `json:"message"` + ReqID string `json:"request_id"` +} + +type errorBody struct { + Error apiError `json:"error"` +} + +type App struct { + db *pgxpool.Pool + log *slog.Logger + ready atomic.Bool + sessionTTL time.Duration + idempotencyTTL time.Duration + store *Store + limits limits + uploads chan struct{} + fault string +} + +func NewApp(db *pgxpool.Pool, log *slog.Logger, sessionTTL, idempotencyTTL time.Duration, store *Store, lim limits) *App { + a := &App{db: db, log: log, sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, store: store, limits: lim, uploads: make(chan struct{}, lim.uploadSlots), fault: os.Getenv("FAULT_POINT")} + a.ready.Store(true) + return a +} + +func (a *App) SetReady(v bool) { + a.ready.Store(v) +} + +func newUUID() string { + var b [16]byte + _, _ = rand.Read(b[:]) + b[6] = (b[6] & 0x0f) | 0x40 + b[8] = (b[8] & 0x3f) | 0x80 + return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]) +} + +func nowUTC() time.Time { + return time.Now().UTC().Truncate(time.Microsecond) +} + +func formatTime(t time.Time) string { + return t.UTC().Format("2006-01-02T15:04:05.000000Z") +} + +func parseTime(s string) (time.Time, error) { + return time.Parse("2006-01-02T15:04:05.000000Z", s) +} + +type statusWriter struct { + http.ResponseWriter + status int +} + +func (w *statusWriter) WriteHeader(code int) { + w.status = code + w.ResponseWriter.WriteHeader(code) +} + +type ctxKey int + +const ( + reqIDKey ctxKey = 1 + routeKey ctxKey = 2 +) + +type routeHolder struct{ route string } + +func (a *App) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + reqID := newUUID() + w.Header().Set("X-Request-Id", reqID) + w.Header().Set("X-Instance-Id", instanceID) + sw := &statusWriter{ResponseWriter: w, status: 200} + health := strings.HasPrefix(r.URL.Path, "/health/") + rt := &routeHolder{route: "unmatched"} + ctx := context.WithValue(context.WithValue(r.Context(), reqIDKey, reqID), routeKey, rt) + var span trace.Span + if !health { + ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.HeaderCarrier(r.Header)) + ctx, span = tracer.Start(ctx, r.Method, trace.WithSpanKind(trace.SpanKindServer)) + span.SetAttributes(attribute.String("app.request_id", reqID)) + } + defer func() { + if rec := recover(); rec != nil { + a.writeErr(sw, reqID, 500, "internal_error", "internal error") + a.log.ErrorContext(ctx, "panic", "request_id", reqID, "err", fmt.Sprint(rec)) + } + elapsed := time.Since(start) + level := slog.LevelInfo + if health { + level = slog.LevelDebug + } else { + class := fmt.Sprintf("%dxx", sw.status/100) + attrs := metric.WithAttributes( + attribute.String("http.request.method", r.Method), + attribute.String("http.route", rt.route), + attribute.String("http.response.status_class", class), + ) + httpRequests.Add(ctx, 1, attrs) + httpDuration.Record(ctx, elapsed.Seconds(), attrs) + span.SetName(r.Method + " " + rt.route) + span.SetAttributes(attribute.String("http.request.method", r.Method), attribute.String("http.route", rt.route), + attribute.Int("http.response.status_code", sw.status)) + if sw.status >= 500 { + span.SetStatus(codes.Error, http.StatusText(sw.status)) + } + span.End() + } + a.log.Log(ctx, level, "request", + "method", r.Method, + "path", r.URL.Path, + "status", sw.status, + "duration_ms", elapsed.Milliseconds(), + "request_id", reqID, + ) + }() + next.ServeHTTP(sw, r.WithContext(ctx)) + }) +} + +func reqIDFrom(r *http.Request) string { + if v, ok := r.Context().Value(reqIDKey).(string); ok { + return v + } + return "" +} + +func (a *App) writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + _ = enc.Encode(v) +} + +func (a *App) writeErr(w http.ResponseWriter, reqID string, status int, code, msg string) { + a.writeJSON(w, status, errorBody{Error: apiError{Code: code, Message: msg, ReqID: reqID}}) +} + +func (a *App) writeUnauthorized(w http.ResponseWriter, r *http.Request, realm string) { + reqID := reqIDFrom(r) + if realm == "Basic" { + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + } else { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + } + a.writeErr(w, reqID, 401, "unauthorized", "unauthorized") +} + +func (a *App) Routes() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /health/live", a.handleLive) + mux.HandleFunc("GET /health/ready", a.handleReady) + mux.HandleFunc("POST /api/v1/users", a.handleCreateUser) + mux.HandleFunc("GET /api/v1/users/me", a.handleMe) + mux.HandleFunc("POST /api/v1/chats", a.handleCreateChat) + mux.HandleFunc("GET /api/v1/chats", a.handleListChats) + mux.HandleFunc("GET /api/v1/chats/{chat_id}", a.handleGetChat) + mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", a.handleCreateMessage) + mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", a.handleListMessages) + mux.HandleFunc("POST /api/v1/chats/{chat_id}/attachments", a.handleUploadAttachment) + mux.HandleFunc("GET /api/v1/attachments/{attachment_id}", a.handleGetAttachment) + mux.HandleFunc("GET /api/v1/attachments/{attachment_id}/content", a.handleDownloadAttachment) + mux.HandleFunc("POST /api/v1/auth/sessions", a.handleCreateSession) + mux.HandleFunc("DELETE /api/v1/auth/sessions/current", a.handleRevokeSession) + return a.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, pattern := mux.Handler(r) + if pattern == "" { + a.writeErr(w, reqIDFrom(r), 404, "not_found", "not found") + return + } + if rt, ok := r.Context().Value(routeKey).(*routeHolder); ok { + _, rt.route, _ = strings.Cut(pattern, " ") + } + mux.ServeHTTP(w, r) + })) +} + +func (a *App) handleLive(w http.ResponseWriter, r *http.Request) { + a.writeJSON(w, 200, map[string]string{"status": "ok"}) +} + +func (a *App) handleReady(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + if !a.ready.Load() { + a.writeErr(w, reqID, 503, "unavailable", "shutting down") + return + } + ctx, cancel := dbCtx(r.Context()) + defer cancel() + if err := a.db.Ping(ctx); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 200, map[string]string{"status": "ok"}) +} + +func (a *App) readJSON(w http.ResponseWriter, r *http.Request, dst any) bool { + reqID := reqIDFrom(r) + ct := r.Header.Get("Content-Type") + media := strings.ToLower(strings.TrimSpace(strings.Split(ct, ";")[0])) + if media != "application/json" { + a.writeErr(w, reqID, 415, "unsupported_media_type", "Content-Type must be application/json") + return false + } + r.Body = http.MaxBytesReader(w, r.Body, 1<<20) + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + if _, ok := err.(*http.MaxBytesError); ok { + a.writeErr(w, reqID, 413, "payload_too_large", "body too large") + return false + } + a.writeErr(w, reqID, 400, "invalid_request", "invalid json") + return false + } + var extra json.RawMessage + if err := dec.Decode(&extra); err != io.EOF { + a.writeErr(w, reqID, 400, "invalid_request", "invalid json") + return false + } + return true +} + +func isBlank(s string) bool { + return strings.TrimSpace(s) == "" +} + +func isUUID(s string) bool { + if len(s) != 36 { + return false + } + for i, c := range s { + switch i { + case 8, 13, 18, 23: + if c != '-' { + return false + } + default: + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { + return false + } + } + } + raw := strings.ReplaceAll(s, "-", "") + _, err := hex.DecodeString(raw) + return err == nil +} + +func isPgCode(err error, code string) bool { + var pgErr *pgconn.PgError + return errors.As(err, &pgErr) && pgErr.Code == code +} + +func userJSON(u *User) map[string]any { + return map[string]any{ + "id": u.ID, + "username": u.Username, + "display_name": u.DisplayName, + "created_at": formatTime(u.CreatedAt), + } +} + +func (a *App) authUser(w http.ResponseWriter, r *http.Request) (*User, string) { + token, ok := parseBearer(r.Header.Get("Authorization")) + if !ok { + a.writeUnauthorized(w, r, "Bearer") + return nil, "" + } + hash, hashHex := tokenHash(token) + + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var u User + var expiresAt time.Time + err := a.db.QueryRow(ctx, ` + select u.id::text, u.username, u.display_name, u.password_hash, u.created_at, s.expires_at + from sessions s join users u on u.id = s.user_id + where s.token_hash = $1 and s.expires_at > now()`, hash).Scan( + &u.ID, &u.Username, &u.DisplayName, &u.PasswordHash, &u.CreatedAt, &expiresAt) + if errors.Is(err, pgx.ErrNoRows) { + a.writeUnauthorized(w, r, "Bearer") + return nil, "" + } + if err != nil { + a.writeErr(w, reqIDFrom(r), 503, "unavailable", "database unavailable") + return nil, "" + } + return &u, hashHex +} + +func (a *App) handleCreateUser(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + var body struct { + Username *string `json:"username"` + DisplayName *string `json:"display_name"` + Password *string `json:"password"` + } + if !a.readJSON(w, r, &body) { + return + } + if body.Username == nil || body.DisplayName == nil || body.Password == nil { + a.writeErr(w, reqID, 400, "invalid_request", "username, display_name and password required") + return + } + username := *body.Username + display := *body.DisplayName + password := *body.Password + if !usernameRe.MatchString(username) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid username") + return + } + n := utf8.RuneCountInString(display) + if n < 1 || n > 100 || isBlank(display) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid display_name") + return + } + if !validPassword(password) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid password") + return + } + ph, err := hashPassword(password) + if err != nil { + a.writeErr(w, reqID, 500, "internal_error", "internal error") + return + } + id := newUUID() + createdAt := nowUTC() + ctx, cancel := dbCtx(r.Context()) + defer cancel() + _, err = a.db.Exec(ctx, ` + insert into users(id, username, display_name, password_hash, created_at) + values($1::uuid, $2, $3, $4, $5)`, id, username, display, ph, createdAt) + if isPgCode(err, "23505") { + a.writeErr(w, reqID, 409, "conflict", "username taken") + return + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 201, map[string]any{ + "id": id, + "username": username, + "display_name": display, + "created_at": formatTime(createdAt), + }) +} + +func (a *App) handleCreateSession(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + username, password, ok := parseBasic(r.Header.Get("Authorization")) + if !ok { + a.writeUnauthorized(w, r, "Basic") + return + } + if !usernameRe.MatchString(username) || !validPassword(password) { + a.writeUnauthorized(w, r, "Basic") + return + } + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var u User + err := a.db.QueryRow(ctx, ` + select id::text, username, display_name, password_hash, created_at + from users where username = $1`, username).Scan( + &u.ID, &u.Username, &u.DisplayName, &u.PasswordHash, &u.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + a.writeUnauthorized(w, r, "Basic") + return + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + match, err := checkPassword(u.PasswordHash, password) + if err != nil || !match { + a.writeUnauthorized(w, r, "Basic") + return + } + token, hash, _, err := newToken() + if err != nil { + a.writeErr(w, reqID, 500, "internal_error", "internal error") + return + } + createdAt := nowUTC() + expiresAt := createdAt.Add(a.sessionTTL) + _, err = a.db.Exec(ctx, ` + insert into sessions(token_hash, user_id, created_at, expires_at) + values($1, $2::uuid, $3, $4)`, hash, u.ID, createdAt, expiresAt) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 201, map[string]any{ + "token": token, + "token_type": "Bearer", + "expires_at": formatTime(expiresAt), + "user": userJSON(&u), + }) +} + +func (a *App) handleRevokeSession(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + token, ok := parseBearer(r.Header.Get("Authorization")) + if !ok { + a.writeUnauthorized(w, r, "Bearer") + return + } + hash, _ := tokenHash(token) + ctx, cancel := dbCtx(r.Context()) + defer cancel() + tag, err := a.db.Exec(ctx, ` + delete from sessions where token_hash = $1 and expires_at > now()`, hash) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if tag.RowsAffected() == 0 { + a.writeUnauthorized(w, r, "Bearer") + return + } + w.Header().Set("X-Request-Id", reqID) + w.WriteHeader(204) +} + +func (a *App) handleMe(w http.ResponseWriter, r *http.Request) { + u, _ := a.authUser(w, r) + if u == nil { + return + } + a.writeJSON(w, 200, userJSON(u)) +} + +func (a *App) handleCreateChat(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + var body struct { + Title *string `json:"title"` + MemberIDs *[]string `json:"member_ids"` + } + if !a.readJSON(w, r, &body) { + return + } + if body.Title == nil || body.MemberIDs == nil { + a.writeErr(w, reqID, 400, "invalid_request", "title and member_ids required") + return + } + title := *body.Title + n := utf8.RuneCountInString(title) + if n < 1 || n > 100 || isBlank(title) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid title") + return + } + members := *body.MemberIDs + if len(members) < 1 || len(members) > 100 { + a.writeErr(w, reqID, 400, "invalid_request", "invalid member_ids") + return + } + seen := make(map[string]struct{}, len(members)) + hasCaller := false + for _, id := range members { + if !isUUID(id) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid member id") + return + } + if _, dup := seen[id]; dup { + a.writeErr(w, reqID, 400, "invalid_request", "duplicate member") + return + } + seen[id] = struct{}{} + if id == u.ID { + hasCaller = true + } + } + if !hasCaller { + a.writeErr(w, reqID, 400, "invalid_request", "caller must be member") + return + } + + chatID := newUUID() + createdAt := nowUTC() + ctx, cancel := dbCtx(r.Context()) + defer cancel() + tx, err := a.db.Begin(ctx) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + defer tx.Rollback(ctx) + + _, err = tx.Exec(ctx, `insert into chats(id, title, created_at) values($1::uuid, $2, $3)`, + chatID, title, createdAt) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + for _, mid := range members { + _, err = tx.Exec(ctx, `insert into chat_members(chat_id, user_id) values($1::uuid, $2::uuid)`, + chatID, mid) + if isPgCode(err, "23503") { + a.writeErr(w, reqID, 400, "invalid_request", "unknown member") + return + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + } + if err := tx.Commit(ctx); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 201, map[string]any{ + "id": chatID, + "title": title, + "member_ids": members, + "created_at": formatTime(createdAt), + }) +} + +func encodeCursor(parts ...string) string { + return base64.RawURLEncoding.EncodeToString([]byte(strings.Join(parts, "|"))) +} + +func decodeCursor(s string) ([]string, error) { + b, err := base64.RawURLEncoding.DecodeString(s) + if err != nil { + return nil, err + } + return strings.Split(string(b), "|"), nil +} + +func parseLimit(r *http.Request) (int, error) { + q := r.URL.Query().Get("limit") + if q == "" { + return 50, nil + } + n, err := strconv.Atoi(q) + if err != nil || n < 1 || n > 100 { + return 0, fmt.Errorf("bad limit") + } + return n, nil +} + +func (a *App) loadChatMembers(ctx context.Context, chatID string) ([]string, error) { + ctx, cancel := dbCtx(ctx) + defer cancel() + rows, err := a.db.Query(ctx, ` + select user_id::text from chat_members where chat_id = $1::uuid order by user_id`, chatID) + if err != nil { + return nil, err + } + defer rows.Close() + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + return ids, rows.Err() +} + +func (a *App) isMember(ctx context.Context, chatID, userID string) (bool, error) { + ctx, cancel := dbCtx(ctx) + defer cancel() + var ok bool + err := a.db.QueryRow(ctx, ` + select exists( + select 1 from chat_members where chat_id = $1::uuid and user_id = $2::uuid + )`, chatID, userID).Scan(&ok) + return ok, err +} + +func (a *App) handleListChats(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + limit, err := parseLimit(r) + if err != nil { + a.writeErr(w, reqID, 400, "invalid_request", "invalid limit") + return + } + var afterTS *time.Time + var afterID string + if cur := r.URL.Query().Get("cursor"); cur != "" { + parts, err := decodeCursor(cur) + if err != nil || len(parts) != 3 { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + if parts[0] != u.ID { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + ts, err := parseTime(parts[1]) + if err != nil || !isUUID(parts[2]) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + afterTS = &ts + afterID = parts[2] + } + + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var rows pgx.Rows + if afterTS == nil { + rows, err = a.db.Query(ctx, ` + select c.id::text, c.title, c.created_at + from chats c + join chat_members m on m.chat_id = c.id + where m.user_id = $1::uuid + order by c.created_at, c.id + limit $2`, u.ID, limit+1) + } else { + rows, err = a.db.Query(ctx, ` + select c.id::text, c.title, c.created_at + from chats c + join chat_members m on m.chat_id = c.id + where m.user_id = $1::uuid + and (c.created_at, c.id) > ($2, $3::uuid) + order by c.created_at, c.id + limit $4`, u.ID, *afterTS, afterID, limit+1) + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + defer rows.Close() + + type row struct { + id, title string + created time.Time + } + var list []row + for rows.Next() { + var x row + if err := rows.Scan(&x.id, &x.title, &x.created); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + list = append(list, x) + } + if err := rows.Err(); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + hasMore := len(list) > limit + if hasMore { + list = list[:limit] + } + out := make([]map[string]any, 0, len(list)) + for _, c := range list { + members, err := a.loadChatMembers(r.Context(), c.id) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + out = append(out, map[string]any{ + "id": c.id, + "title": c.title, + "member_ids": members, + "created_at": formatTime(c.created), + }) + } + var next any = nil + if hasMore && len(list) > 0 { + last := list[len(list)-1] + next = encodeCursor(u.ID, formatTime(last.created), last.id) + } + a.writeJSON(w, 200, map[string]any{"items": out, "next_cursor": next}) +} + +func (a *App) handleGetChat(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + chatID := r.PathValue("chat_id") + if !isUUID(chatID) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id") + return + } + ok, err := a.isMember(r.Context(), chatID, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if !ok { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var title string + var createdAt time.Time + err = a.db.QueryRow(ctx, `select title, created_at from chats where id = $1::uuid`, chatID). + Scan(&title, &createdAt) + if errors.Is(err, pgx.ErrNoRows) { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + members, err := a.loadChatMembers(r.Context(), chatID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 200, map[string]any{ + "id": chatID, + "title": title, + "member_ids": members, + "created_at": formatTime(createdAt), + }) +} + +func (a *App) handleCreateMessage(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + chatID := r.PathValue("chat_id") + if !isUUID(chatID) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id") + return + } + key := r.Header.Get("Idempotency-Key") + if !idempotencyKeyRe.MatchString(key) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid Idempotency-Key") + return + } + ok, err := a.isMember(r.Context(), chatID, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if !ok { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + var body struct { + Text *string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + } + if !a.readJSON(w, r, &body) { + return + } + if body.Text == nil { + a.writeErr(w, reqID, 400, "invalid_request", "text required") + return + } + text := *body.Text + attachmentIDs := body.AttachmentIDs + if attachmentIDs == nil { + attachmentIDs = []string{} + } + if len(attachmentIDs) > 10 { + a.writeErr(w, reqID, 400, "invalid_request", "too many attachments") + return + } + seen := make(map[string]bool, len(attachmentIDs)) + for _, id := range attachmentIDs { + if !isUUID(id) || seen[id] { + a.writeErr(w, reqID, 400, "invalid_request", "invalid attachment_ids") + return + } + seen[id] = true + } + n := utf8.RuneCountInString(text) + if n > 4000 || (len(attachmentIDs) == 0 && (n < 1 || isBlank(text))) || (n > 0 && isBlank(text)) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid text") + return + } + if len(attachmentIDs) > 0 { + var found, failed int + cctx, ccancel := dbCtx(r.Context()) + err := a.db.QueryRow(cctx, ` + select count(*), count(*) filter (where status = 'failed') + from attachments where id = any($1::uuid[]) and chat_id = $2::uuid`, attachmentIDs, chatID).Scan(&found, &failed) + ccancel() + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if found != len(attachmentIDs) { + a.writeErr(w, reqID, 404, "not_found", "attachment not found") + return + } + if failed > 0 { + a.writeErr(w, reqID, 409, "conflict", "attachment processing failed") + return + } + } + hashInput := text + if len(attachmentIDs) > 0 { + hashInput += "\x00" + strings.Join(attachmentIDs, ",") + } + sum := sha256.Sum256([]byte(hashInput)) + bodyHash := hex.EncodeToString(sum[:]) + path := r.URL.Path + ctx, cancel := dbCtx(r.Context()) + defer cancel() + tx, err := a.db.Begin(ctx) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + defer tx.Rollback(ctx) + + if _, err := tx.Exec(ctx, ` + delete from idempotency_keys + where user_id=$1::uuid and method='POST' and path=$2 and key=$3 + and created_at < $4`, + u.ID, path, key, time.Now().Add(-a.idempotencyTTL)); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + msgID := newUUID() + createdAt := nowUTC() + resp := map[string]any{ + "id": msgID, + "chat_id": chatID, + "sender_id": u.ID, + "text": text, + "created_at": formatTime(createdAt), + "attachment_ids": attachmentIDs, + } + respJSON, _ := json.Marshal(resp) + tag, err := tx.Exec(ctx, ` + insert into idempotency_keys(user_id, method, path, key, body_hash, response, created_at) + values($1::uuid, 'POST', $2, $3, $4, $5, $6) + on conflict do nothing`, + u.ID, path, key, bodyHash, respJSON, createdAt) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if tag.RowsAffected() == 0 { + var storedHash string + var stored []byte + err := tx.QueryRow(ctx, ` + select body_hash, response from idempotency_keys + where user_id=$1::uuid and method='POST' and path=$2 and key=$3`, + u.ID, path, key).Scan(&storedHash, &stored) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if storedHash != bodyHash { + a.writeErr(w, reqID, 409, "conflict", "idempotency key reused with different body") + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(201) + _, _ = w.Write(append(stored, '\n')) + return + } + if _, err := tx.Exec(ctx, ` + insert into messages(id, chat_id, sender_id, text, created_at) + values($1::uuid, $2::uuid, $3::uuid, $4, $5)`, + msgID, chatID, u.ID, text, createdAt); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if len(attachmentIDs) > 0 { + if _, err := tx.Exec(ctx, ` + insert into message_attachments(message_id, attachment_id, position) + select $1::uuid, t.id, t.ord - 1 + from unnest($2::uuid[]) with ordinality as t(id, ord)`, + msgID, attachmentIDs); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + } + if err := tx.Commit(ctx); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 201, resp) +} + +func (a *App) handleListMessages(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + chatID := r.PathValue("chat_id") + if !isUUID(chatID) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id") + return + } + ok, err := a.isMember(r.Context(), chatID, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if !ok { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + limit, err := parseLimit(r) + if err != nil { + a.writeErr(w, reqID, 400, "invalid_request", "invalid limit") + return + } + var afterTS *time.Time + var afterID string + if cur := r.URL.Query().Get("cursor"); cur != "" { + parts, err := decodeCursor(cur) + if err != nil || len(parts) != 3 { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + if parts[0] != chatID { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + ts, err := parseTime(parts[1]) + if err != nil || !isUUID(parts[2]) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + afterTS = &ts + afterID = parts[2] + } + + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var rows pgx.Rows + if afterTS == nil { + rows, err = a.db.Query(ctx, ` + select id::text, chat_id::text, sender_id::text, text, created_at + from messages + where chat_id = $1::uuid + order by created_at, id + limit $2`, chatID, limit+1) + } else { + rows, err = a.db.Query(ctx, ` + select id::text, chat_id::text, sender_id::text, text, created_at + from messages + where chat_id = $1::uuid + and (created_at, id) > ($2, $3::uuid) + order by created_at, id + limit $4`, chatID, *afterTS, afterID, limit+1) + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + defer rows.Close() + + type row struct { + id, chatID, senderID, text string + created time.Time + } + var list []row + for rows.Next() { + var x row + if err := rows.Scan(&x.id, &x.chatID, &x.senderID, &x.text, &x.created); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + list = append(list, x) + } + if err := rows.Err(); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + hasMore := len(list) > limit + if hasMore { + list = list[:limit] + } + attachments := map[string][]string{} + if len(list) > 0 { + ids := make([]string, len(list)) + for i, m := range list { + ids[i] = m.id + } + arows, err := a.db.Query(ctx, ` + select message_id::text, attachment_id::text + from message_attachments + where message_id = any($1::uuid[]) + order by message_id, position`, ids) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + for arows.Next() { + var mid, aid string + if err := arows.Scan(&mid, &aid); err != nil { + arows.Close() + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + attachments[mid] = append(attachments[mid], aid) + } + arows.Close() + if err := arows.Err(); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + } + out := make([]map[string]any, 0, len(list)) + for _, m := range list { + ids := attachments[m.id] + if ids == nil { + ids = []string{} + } + out = append(out, map[string]any{ + "id": m.id, + "chat_id": m.chatID, + "sender_id": m.senderID, + "text": m.text, + "created_at": formatTime(m.created), + "attachment_ids": ids, + }) + } + var next any = nil + if hasMore && len(list) > 0 { + last := list[len(list)-1] + next = encodeCursor(chatID, formatTime(last.created), last.id) + } + a.writeJSON(w, 200, map[string]any{"items": out, "next_cursor": next}) +} + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} diff --git a/imaging.go b/imaging.go new file mode 100644 index 0000000..6ccc5f2 --- /dev/null +++ b/imaging.go @@ -0,0 +1,176 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + "image/draw" + "image/jpeg" + _ "image/png" + + "github.com/gen2brain/webp" + "github.com/rwcarlsen/goexif/exif" + xdraw "golang.org/x/image/draw" +) + +const ( + jpegQuality = 85 + webpQuality = 80 + webpMethod = 4 +) + +var errInvalidImage = errors.New("invalid image") + +type variantSpec struct { + name, contentType, ext string + box int + crop bool +} + +var variantSpecs = []variantSpec{ + {"thumbnail", "image/jpeg", "jpg", 256, false}, + {"crop", "image/jpeg", "jpg", 128, true}, + {"optimized", "image/webp", "webp", 1280, false}, +} + +type rendered struct { + spec variantSpec + data []byte + width, height int + sha256 string +} + +func renderVariants(ctx context.Context, data []byte, maxPixels int64) ([]rendered, error) { + cfg, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || cfg.Width < 1 || cfg.Height < 1 { + return nil, errInvalidImage + } + if int64(cfg.Width)*int64(cfg.Height) > maxPixels { + return nil, errInvalidImage + } + src, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return nil, errInvalidImage + } + orient := 1 + if format == "jpeg" { + orient = orientation(data) + } + src = shrink(src, 1280) + var out []rendered + for _, spec := range variantSpecs { + if err := ctx.Err(); err != nil { + return nil, err + } + img := render(src, cfg.Width, cfg.Height, orient, spec) + var buf bytes.Buffer + if spec.contentType == "image/webp" { + err = webp.Encode(&buf, img, webp.Options{Quality: webpQuality, Method: webpMethod}) + } else { + err = jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality}) + } + if err != nil { + return nil, err + } + sum := sha256.Sum256(buf.Bytes()) + b := img.Bounds() + out = append(out, rendered{spec: spec, data: buf.Bytes(), width: b.Dx(), height: b.Dy(), sha256: hex.EncodeToString(sum[:])}) + } + return out, nil +} + +func orientation(data []byte) int { + x, err := exif.Decode(bytes.NewReader(data)) + if err != nil { + return 1 + } + tag, err := x.Get(exif.Orientation) + if err != nil { + return 1 + } + v, err := tag.Int(0) + if err != nil || v < 1 || v > 8 { + return 1 + } + return v +} + +func fit(w, h, box int) (int, int) { + if w <= box && h <= box { + return w, h + } + if w >= h { + return box, max(1, h*box/w) + } + return max(1, w*box/h), box +} + +func shrink(src image.Image, limit int) image.Image { + for { + b := src.Bounds() + if max(b.Dx(), b.Dy())/2 < limit { + return src + } + dst := image.NewRGBA(image.Rect(0, 0, max(1, b.Dx()/2), max(1, b.Dy()/2))) + xdraw.ApproxBiLinear.Scale(dst, dst.Bounds(), src, b, xdraw.Src, nil) + src = dst + } +} + +func render(src image.Image, sw, sh, orient int, spec variantSpec) *image.RGBA { + b := src.Bounds() + rect := b + var tw, th int + if spec.crop { + side := min(b.Dx(), b.Dy()) + x0, y0 := b.Min.X+(b.Dx()-side)/2, b.Min.Y+(b.Dy()-side)/2 + rect = image.Rect(x0, y0, x0+side, y0+side) + tw, th = spec.box, spec.box + } else if orient >= 5 { + th, tw = fit(sh, sw, spec.box) + } else { + tw, th = fit(sw, sh, spec.box) + } + dst := image.NewRGBA(image.Rect(0, 0, tw, th)) + draw.Draw(dst, dst.Bounds(), image.White, image.Point{}, draw.Src) + xdraw.CatmullRom.Scale(dst, dst.Bounds(), src, rect, xdraw.Over, nil) + return orientImage(dst, orient) +} + +func orientImage(img *image.RGBA, o int) *image.RGBA { + if o <= 1 { + return img + } + w, h := img.Bounds().Dx(), img.Bounds().Dy() + dw, dh := w, h + if o >= 5 { + dw, dh = h, w + } + dst := image.NewRGBA(image.Rect(0, 0, dw, dh)) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + var nx, ny int + switch o { + case 2: + nx, ny = w-1-x, y + case 3: + nx, ny = w-1-x, h-1-y + case 4: + nx, ny = x, h-1-y + case 5: + nx, ny = y, x + case 6: + nx, ny = h-1-y, x + case 7: + nx, ny = h-1-y, w-1-x + case 8: + nx, ny = y, w-1-x + } + copy(dst.Pix[dst.PixOffset(nx, ny):dst.PixOffset(nx, ny)+4], img.Pix[img.PixOffset(x, y):img.PixOffset(x, y)+4]) + } + } + return dst +} diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..2715d9e --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,1317 @@ +{ + "uid": "messenger-observability", + "title": "Messenger observability", + "tags": [ + "messenger", + "lab6" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "SLI / SLO", + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Availability SLI (1 - 5xx share), selected range", + "description": "SLO: 99% of API requests are not 5xx over 30 days. Query: 1 - 5xx/all from http_server_requests_total.", + "gridPos": { + "x": 0, + "y": 1, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.99 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value" + }, + "targets": [ + { + "refId": "A", + "expr": "1 - (sum(increase(http_server_requests_total{http_response_status_class=\"5xx\"}[$__range])) or vector(0)) / sum(increase(http_server_requests_total[$__range]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "instant": true + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Ready within 10s SLI, selected range", + "description": "SLO: 95% of attachments reach ready within 10 s over 30 days. Bucket le=10 of job_time_to_ready_seconds.", + "gridPos": { + "x": 6, + "y": 1, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.95 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(increase(job_time_to_ready_seconds_bucket{le=\"10\"}[$__range])) / sum(increase(job_time_to_ready_seconds_count[$__range]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "instant": true + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Firing alerts", + "description": "Prometheus alert rules in infra/prometheus/rules/alerts.yml", + "gridPos": { + "x": 12, + "y": 1, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value" + }, + "targets": [ + { + "refId": "A", + "expr": "count(ALERTS{alertstate=\"firing\"}) or vector(0)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "instant": true + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Telemetry export errors, 5m", + "description": "Failed OTLP exports counted in the application SDK", + "gridPos": { + "x": 18, + "y": 1, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(increase(telemetry_export_errors_total[5m])) or vector(0)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "instant": true + } + ] + }, + { + "id": 6, + "type": "row", + "title": "API", + "gridPos": { + "x": 0, + "y": 5, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 7, + "type": "timeseries", + "title": "Requests per second by route", + "description": "Counter http_server_requests_total, rate over the dashboard interval", + "gridPos": { + "x": 0, + "y": 6, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (http_route) (rate(http_server_requests_total[$__rate_interval]))", + "legendFormat": "{{http_route}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "Error ratio (5xx / all)", + "description": "Numerator: 5xx responses, denominator: all API requests (health checks are not recorded). 4xx are client errors and are not counted.", + "gridPos": { + "x": 8, + "y": 6, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "thresholdsStyle": { + "mode": "line" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.05 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "(sum(rate(http_server_requests_total{http_response_status_class=\"5xx\"}[$__rate_interval])) or vector(0)) / sum(rate(http_server_requests_total[$__rate_interval]))", + "legendFormat": "5xx share", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 9, + "type": "timeseries", + "title": "Latency p50 / p95 / p99", + "description": "histogram_quantile over summed http_server_request_duration_seconds_bucket of all replicas", + "gridPos": { + "x": 16, + "y": 6, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 10, + "type": "row", + "title": "Jobs and queue", + "gridPos": { + "x": 0, + "y": 14, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 11, + "type": "timeseries", + "title": "Jobs per second by outcome", + "description": "jobs_processed_total: ready, retry, failed, skipped, lease_lost", + "gridPos": { + "x": 0, + "y": 15, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (outcome) (rate(jobs_processed_total[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "Job duration p50 / p95 / p99", + "description": "job_processing_duration_seconds_bucket, worker processing time of one attempt", + "gridPos": { + "x": 8, + "y": 15, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(job_processing_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(job_processing_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(job_processing_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 13, + "type": "timeseries", + "title": "Time to ready p50 / p95 / p99", + "description": "job_time_to_ready_seconds: from durable acceptance to ready commit", + "gridPos": { + "x": 16, + "y": 15, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(job_time_to_ready_seconds_bucket[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(job_time_to_ready_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(job_time_to_ready_seconds_bucket[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 14, + "type": "timeseries", + "title": "Queue depth", + "description": "Jobs in status queued (max across API replicas, they report the same PostgreSQL value)", + "gridPos": { + "x": 0, + "y": 23, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "max by (queue) (queue_depth)", + "legendFormat": "{{queue}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 15, + "type": "timeseries", + "title": "Oldest queued job age", + "description": "Seconds since the oldest available queued job became due; alert QueueLagGrowing fires above 20 s", + "gridPos": { + "x": 8, + "y": 23, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "thresholdsStyle": { + "mode": "line" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 20 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "max by (queue) (queue_oldest_job_age_seconds)", + "legendFormat": "{{queue}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 16, + "type": "timeseries", + "title": "Worker saturation", + "description": "worker_busy jobs in flight versus worker_capacity (WORKER_CONCURRENCY)", + "gridPos": { + "x": 16, + "y": 23, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum(worker_busy)", + "legendFormat": "busy", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum(worker_capacity)", + "legendFormat": "capacity", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 17, + "type": "row", + "title": "Saturation", + "gridPos": { + "x": 0, + "y": 31, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 18, + "type": "timeseries", + "title": "DB pool connections", + "description": "db_pool_connections by state, db_pool_max_connections limit", + "gridPos": { + "x": 0, + "y": 32, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (instance) (db_pool_connections{state=\"used\",job=\"messenger-api\"})", + "legendFormat": "used {{instance}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum(db_pool_connections{state=\"used\",job=\"messenger-worker\"})", + "legendFormat": "used worker", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "max(db_pool_max_connections)", + "legendFormat": "max", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 19, + "type": "timeseries", + "title": "DB pool acquire wait", + "description": "Time and count of acquires that had to wait for a free connection", + "gridPos": { + "x": 8, + "y": 32, + "w": 8, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (job) (rate(db_pool_acquire_wait_seconds_total[$__rate_interval]))", + "legendFormat": "wait s/s {{job}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum by (job) (rate(db_pool_empty_acquires_total[$__rate_interval]))", + "legendFormat": "waited acquires/s {{job}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 20, + "type": "timeseries", + "title": "Process CPU", + "description": "CPU seconds per second of the application processes", + "gridPos": { + "x": 16, + "y": 32, + "w": 4, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (job) (rate(process_cpu_time_seconds_total[$__rate_interval]))", + "legendFormat": "{{job}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 21, + "type": "timeseries", + "title": "Process RSS", + "description": "Resident memory of the application processes", + "gridPos": { + "x": 20, + "y": 32, + "w": 4, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (job) (process_memory_usage_bytes)", + "legendFormat": "{{job}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 22, + "type": "row", + "title": "Telemetry pipeline", + "gridPos": { + "x": 0, + "y": 40, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 23, + "type": "timeseries", + "title": "Collector exporter queue", + "description": "Bounded sending queue of each Collector exporter (capacity 100 batches)", + "gridPos": { + "x": 0, + "y": 41, + "w": 8, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (exporter) (otelcol_exporter_queue_size)", + "legendFormat": "{{exporter}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 24, + "type": "timeseries", + "title": "Collector send failures", + "description": "Data the Collector could not deliver to a storage backend", + "gridPos": { + "x": 8, + "y": 41, + "w": 8, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum(rate(otelcol_exporter_send_failed_spans[$__rate_interval]))", + "legendFormat": "spans", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum(rate(otelcol_exporter_send_failed_metric_points[$__rate_interval]))", + "legendFormat": "metric points", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum(rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))", + "legendFormat": "log records", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 25, + "type": "timeseries", + "title": "Collector memory", + "description": "Collector RSS, bounded by memory_limiter and the container limit", + "gridPos": { + "x": 16, + "y": 41, + "w": 8, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "otelcol_process_memory_rss", + "legendFormat": "rss", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "id": 26, + "type": "row", + "title": "Logs and traces", + "gridPos": { + "x": 0, + "y": 48, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 27, + "type": "logs", + "title": "Application logs (warn and error)", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "x": 0, + "y": 49, + "w": 12, + "h": 10 + }, + "options": { + "showTime": true, + "wrapLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{service_name=~\"messenger-.*\"} | detected_level=~\"warn|error\"" + } + ] + }, + { + "id": 28, + "type": "table", + "title": "Recent traces with errors", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "x": 12, + "y": 49, + "w": 12, + "h": 10 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "traceql", + "query": "{ status = error }", + "limit": 20, + "tableType": "traces" + } + ] + } + ], + "templating": { + "list": [] + }, + "annotations": { + "list": [] + } +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..2c7add2 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: messenger + type: file + disableDeletion: true + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..1e2a6af --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,34 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: trace_id + matcherType: label + matcherRegex: trace_id + url: "$${__value.raw}" + datasourceUid: tempo + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + filterByTraceID: true + customQuery: true + query: '{service_name=~"messenger-.*"} | trace_id="$${__trace.traceId}"' + serviceMap: + datasourceUid: prometheus diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..1f59a8d --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,47 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + log_level: warn + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +ingester: + wal: + enabled: true + dir: /loki/wal + flush_on_shutdown: true + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + retention_period: 168h + ingestion_rate_mb: 8 + ingestion_burst_size_mb: 16 + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..e844c36 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,77 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 160 + spike_limit_mib: 40 + batch: + timeout: 2s + send_batch_size: 512 + send_batch_max_size: 1024 + +exporters: + otlphttp/prometheus: + endpoint: http://prometheus:9090/api/v1/otlp + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 100 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 20s + otlphttp/loki: + endpoint: http://loki:3100/otlp + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 100 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 20s + otlp/tempo: + endpoint: tempo:4317 + timeout: 5s + tls: + insecure: true + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 100 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 20s + +service: + telemetry: + logs: + level: warn + metrics: + address: 0.0.0.0:8888 + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/tempo] diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..96a7622 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,19 @@ +global: + scrape_interval: 10s + evaluation_interval: 10s + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + +otlp: + promote_resource_attributes: + - service.version + - deployment.environment.name diff --git a/infra/prometheus/rules/alerts.yml b/infra/prometheus/rules/alerts.yml new file mode 100644 index 0000000..94c2c19 --- /dev/null +++ b/infra/prometheus/rules/alerts.yml @@ -0,0 +1,23 @@ +groups: + - name: messenger-slo + rules: + - alert: ApiHighErrorRate + expr: | + sum(rate(http_server_requests_total{http_response_status_class="5xx"}[1m])) + / sum(rate(http_server_requests_total[1m])) > 0.05 + for: 20s + labels: + severity: page + annotations: + summary: "API 5xx ratio above 5% over 1m" + description: "SLI: share of non-5xx business requests; SLO 99% over 30d. Fast-burn check on a 1m window." + runbook: "infra/runbook.md#apihigherrorrate" + - alert: QueueLagGrowing + expr: max(queue_oldest_job_age_seconds{queue="attachments.jobs"}) > 20 + for: 20s + labels: + severity: page + annotations: + summary: "Oldest queued job older than 20s" + description: "SLO: 95% of attachments reach ready within 10s over 30d. A job waiting more than 20s means the worker does not keep up." + runbook: "infra/runbook.md#queuelaggrowing" diff --git a/infra/runbook.md b/infra/runbook.md new file mode 100644 index 0000000..92dc600 --- /dev/null +++ b/infra/runbook.md @@ -0,0 +1,31 @@ +# Runbook + +Grafana: `http://127.0.0.1:3000` (пользователь `admin`, пароль в `secrets/grafana_admin_password`), dashboard «Messenger observability». Состояние alerts: `http://127.0.0.1:9090/alerts`. + +## ApiHighErrorRate + +Условие: доля ответов 5xx за 1 минуту больше 5% в течение 20 с. SLI: доля запросов API без 5xx, SLO 99% за 30 дней. + +1. Dashboard, «Error ratio» и «Requests per second by route»: какие маршруты дают 5xx. Запрос: `sum by (http_route) (rate(http_server_requests_total{http_response_status_class="5xx"}[1m]))`. +2. «Latency p50 / p95 / p99»: рост p99 до 3 с означает таймаут обращения к БД (`dbCtx` ограничивает запрос 3 с). +3. Tempo, запрос `{ resource.service.name = "messenger-api" && status = error }`: открыть trace. Если почти всё время server span занимает один дочерний `db ...` span со статусом error, причина в PostgreSQL, а не в обработчике. +4. «DB pool connections» и «DB pool acquire wait»: все соединения заняты или растёт ожидание подтверждает блокировку или медленную БД. +5. Loki: `{service_name="messenger-api"} | detected_level="error"`; по `trace_id` из лога открыть тот же trace. +6. PostgreSQL: `select pid, state, wait_event_type, left(query, 60) from pg_stat_activity where datname = 'messenger'` и `pg_locks` покажут блокирующую сессию. Завершить её (`pg_terminate_backend`) или устранить причину. +7. Alert разрешается сам, когда доля 5xx за минуту падает ниже порога. + +## QueueLagGrowing + +Условие: возраст самой старой доступной задачи в очереди `attachments.jobs` больше 20 с в течение 20 с. SLO: 95% вложений становятся `ready` за 10 с за 30 дней. + +1. Dashboard, «Queue depth» и «Oldest queued job age»: растут ли оба графика. +2. «Jobs per second by outcome» и «Worker saturation»: если `worker_busy` равен нулю и обработок нет, worker не работает. Если `worker_busy` равен `worker_capacity`, worker перегружен. +3. `docker compose --env-file .env ps worker` и Loki `{service_name="messenger-worker"}`: нет свежих логов означает остановленный процесс. +4. Tempo, `{ name = "attachments.jobs publish" }`: trace зависшей задачи содержит span публикации API и не содержит consumer span worker. +5. Если worker работает, но задачи падают: «Jobs per second by outcome» (`retry`, `failed`), Loki `{service_name="messenger-worker"} | detected_level=~"warn|error"`, trace попыток в одном trace, зависимость (S3, PostgreSQL) в error spans. +6. Запустить worker: `docker compose --env-file .env up -d worker`. Очередь разбирается, alert разрешается, когда возраст падает ниже порога. +7. Задачи в `failed`: `docker compose --env-file .env run --rm --no-deps -T worker failed`, повтор: `... worker replay `. + +## Недоступно хранилище телеметрии + +Приложение продолжает работать. Collector ограничивает память (`memory_limiter`, очередь экспортёра 100 батчей, повтор до 20 с), данные за время недоступности теряются. Потери: `otelcol_exporter_send_failed_*` на dashboard, «Collector send failures»; ошибки экспорта SDK: `telemetry_export_errors_total`. После запуска хранилища: `docker compose --env-file .env up -d prometheus loki tempo`. diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..c05cd63 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,27 @@ +server: + http_listen_port: 3200 + log_level: warn + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +compactor: + compaction: + block_retention: 168h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +usage_report: + reporting_enabled: false diff --git a/lb/haproxy.cfg b/lb/haproxy.cfg new file mode 100644 index 0000000..d63fed5 --- /dev/null +++ b/lb/haproxy.cfg @@ -0,0 +1,38 @@ +global + log stdout format raw local0 + maxconn 2000 + +resolvers docker + nameserver dns 127.0.0.11:53 + accepted_payload_size 8192 + hold valid 1s + hold nx 1s + hold obsolete 1s + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + +defaults + mode http + log global + option httplog + option redispatch 1 + retries 3 + retry-on conn-failure empty-response + timeout connect 1s + timeout client 20s + timeout server 15s + timeout http-request 10s + timeout queue 5s + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr none on-marked-down shutdown-sessions + server-template api 1-8 api:8080 diff --git a/main.go b/main.go new file mode 100644 index 0000000..a29bafd --- /dev/null +++ b/main.go @@ -0,0 +1,214 @@ +package main + +import ( + "context" + "encoding/json" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func main() { + log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})) + mode := "" + if len(os.Args) > 1 { + mode = os.Args[1] + } + if mode == "" || mode == "worker" { + service := "messenger-api" + if mode == "worker" { + service = "messenger-worker" + } + var flush func(context.Context) + log, flush = setupTelemetry(context.Background(), service) + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + flush(ctx) + }() + } + + host := envOr("APP_HOST", "0.0.0.0") + port := envOr("APP_PORT", "8080") + shutdownSec, _ := strconv.Atoi(envOr("SHUTDOWN_TIMEOUT_SECONDS", "5")) + if shutdownSec < 1 { + shutdownSec = 5 + } + ttlSec, _ := strconv.Atoi(envOr("SESSION_TTL_SECONDS", "3600")) + if ttlSec < 1 { + ttlSec = 3600 + } + + idemSec, _ := strconv.Atoi(envOr("IDEMPOTENCY_TTL_SECONDS", "86400")) + if idemSec < 1 { + idemSec = 86400 + } + drainSec, _ := strconv.Atoi(envOr("DRAIN_SECONDS", "3")) + + ctx := context.Background() + if len(os.Args) > 1 && os.Args[1] == "s3-init" { + if err := s3Init(ctx); err != nil { + log.Error("s3 init failed", "err", err.Error()) + os.Exit(1) + } + log.Info("bucket ready") + return + } + db, err := openDB(ctx) + if err != nil { + log.Error("database open failed", "err", err.Error()) + os.Exit(1) + } + defer db.Close() + + if len(os.Args) > 1 && os.Args[1] == "migrate" { + if err := migrate(ctx, db); err != nil { + log.Error("migrate failed", "err", err.Error()) + os.Exit(1) + } + log.Info("migrations applied") + return + } + + store, err := openStore() + if err != nil { + log.Error("s3 open failed", "err", err.Error()) + os.Exit(1) + } + + if len(os.Args) > 1 && (os.Args[1] == "cleanup" || os.Args[1] == "janitor") { + runCleanup(ctx, db, store, log, os.Args[1] == "janitor") + return + } + + maxBytes, _ := strconv.ParseInt(envOr("MAX_UPLOAD_BYTES", "10485760"), 10, 64) + maxPixels, _ := strconv.ParseInt(envOr("MAX_IMAGE_PIXELS", "40000000"), 10, 64) + maxAttempts := envInt("JOB_MAX_ATTEMPTS", 5) + + if len(os.Args) > 1 && (os.Args[1] == "replay" || os.Args[1] == "failed") { + out := json.NewEncoder(os.Stdout) + if os.Args[1] == "failed" { + items, err := listFailed(ctx, db) + if err != nil { + log.Error("failed jobs query failed", "err", err.Error()) + os.Exit(1) + } + for _, item := range items { + _ = out.Encode(item) + } + return + } + if len(os.Args) != 3 || !isUUID(os.Args[2]) { + log.Error("usage: replay ") + os.Exit(2) + } + result, err := replayJob(ctx, db, os.Args[2], maxAttempts) + if err != nil { + log.Error("replay failed", "err", err.Error()) + os.Exit(1) + } + _ = out.Encode(map[string]string{"attachment_id": os.Args[2], "result": result}) + return + } + + broker, err := openBroker() + if err != nil { + log.Error("broker open failed", "err", err.Error()) + os.Exit(1) + } + + if len(os.Args) > 1 && os.Args[1] == "worker" { + timeout := time.Duration(envInt("JOB_TIMEOUT_SECONDS", 60)) * time.Second + owner, _ := os.Hostname() + w := &Worker{ + db: db, store: store, broker: broker, log: log, + owner: owner + "-" + newUUID()[:8], + timeout: timeout, + lease: timeout + 15*time.Second, + backoffBase: time.Duration(envInt("JOB_BACKOFF_BASE_MS", 1000)) * time.Millisecond, + backoffMax: time.Duration(envInt("JOB_BACKOFF_MAX_MS", 30000)) * time.Millisecond, + maxBytes: maxBytes, + maxPixels: maxPixels, + fault: os.Getenv("FAULT_POINT"), + } + runWorker(db, store, broker, log, w, envInt("WORKER_CONCURRENCY", 2)) + return + } + + slots := envInt("UPLOAD_CONCURRENCY", 10) + app := NewApp(db, log, time.Duration(ttlSec)*time.Second, time.Duration(idemSec)*time.Second, store, + limits{maxBytes: maxBytes, maxPixels: maxPixels, uploadSlots: slots, maxAttempts: maxAttempts}) + registerQueueMetrics(db) + relayCtx, stopRelay := context.WithCancel(ctx) + defer stopRelay() + go runRelay(relayCtx, db, broker, log) + + addr := net.JoinHostPort(host, port) + srv := &http.Server{ + Addr: addr, + Handler: app.Routes(), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 15 * time.Second, + IdleTimeout: 30 * time.Second, + } + + go func() { + log.Info("listening", "addr", addr, "env", envOr("APP_ENV", "lab"), "session_ttl_s", ttlSec) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Error("server error", "err", err.Error()) + os.Exit(1) + } + }() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) + sig := <-stop + log.Info("shutdown started", "signal", sig.String()) + app.SetReady(false) + time.Sleep(time.Duration(drainSec) * time.Second) + + shCtx, cancel := context.WithTimeout(context.Background(), time.Duration(shutdownSec)*time.Second) + defer cancel() + if err := srv.Shutdown(shCtx); err != nil { + log.Error("shutdown error", "err", err.Error()) + os.Exit(1) + } + log.Info("shutdown finished") +} + +func runCleanup(ctx context.Context, db *pgxpool.Pool, store *Store, log *slog.Logger, loop bool) { + graceSec, _ := strconv.Atoi(envOr("CLEANUP_GRACE_SECONDS", "600")) + intervalSec, _ := strconv.Atoi(envOr("CLEANUP_INTERVAL_SECONDS", "300")) + retention := time.Duration(envInt("STAGING_RETENTION_SECONDS", 86400)) * time.Second + for { + removed, err := cleanupOrphans(ctx, db, store, time.Duration(graceSec)*time.Second, retention, log) + if err != nil { + log.Error("cleanup failed", "err", err.Error()) + if !loop { + os.Exit(1) + } + } else { + log.Info("cleanup finished", "removed", removed) + } + if !loop { + return + } + time.Sleep(time.Duration(intervalSec) * time.Second) + } +} + +func envInt(key string, def int) int { + v, err := strconv.Atoi(os.Getenv(key)) + if err != nil || v < 1 { + return def + } + return v +} diff --git a/migrate.go b/migrate.go new file mode 100644 index 0000000..d2b953f --- /dev/null +++ b/migrate.go @@ -0,0 +1,84 @@ +package main + +import ( + "context" + "embed" + "fmt" + "sort" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +//go:embed migrations/*.sql +var migrationFS embed.FS + +func migrate(ctx context.Context, pool *pgxpool.Pool) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + conn, err := pool.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + if _, err := conn.Exec(ctx, `select pg_advisory_lock(727001)`); err != nil { + return err + } + defer conn.Exec(context.Background(), `select pg_advisory_unlock(727001)`) + + _, err = conn.Exec(ctx, ` + create table if not exists schema_migrations ( + version text primary key, + applied_at timestamptz not null + )`) + if err != nil { + return fmt.Errorf("schema_migrations: %w", err) + } + + entries, err := migrationFS.ReadDir("migrations") + if err != nil { + return err + } + var names []string + for _, e := range entries { + if !e.IsDir() && strings.HasSuffix(e.Name(), ".sql") { + names = append(names, e.Name()) + } + } + sort.Strings(names) + + for _, name := range names { + var exists bool + err := conn.QueryRow(ctx, `select exists(select 1 from schema_migrations where version=$1)`, name).Scan(&exists) + if err != nil { + return err + } + if exists { + continue + } + + body, err := migrationFS.ReadFile("migrations/" + name) + if err != nil { + return err + } + + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + if _, err := tx.Exec(ctx, string(body)); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("migration %s: %w", name, err) + } + if _, err := tx.Exec(ctx, `insert into schema_migrations(version, applied_at) values($1, now())`, name); err != nil { + _ = tx.Rollback(ctx) + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + } + return nil +} diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 0000000..a50290e --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,38 @@ +create table users ( + id uuid primary key, + username text not null unique, + display_name text not null, + password_hash text not null, + created_at timestamptz not null +); + +create table chats ( + id uuid primary key, + title text not null, + created_at timestamptz not null +); + +create table chat_members ( + chat_id uuid not null references chats(id), + user_id uuid not null references users(id), + primary key (chat_id, user_id) +); + +create index chat_members_user_idx on chat_members (user_id); + +create table messages ( + id uuid primary key, + chat_id uuid not null references chats(id), + sender_id uuid not null references users(id), + text text not null, + created_at timestamptz not null +); + +create index messages_history_idx on messages (chat_id, created_at, id); + +create table sessions ( + token_hash bytea primary key, + user_id uuid not null references users(id) on delete cascade, + created_at timestamptz not null, + expires_at timestamptz not null +); diff --git a/migrations/0002_idempotency.sql b/migrations/0002_idempotency.sql new file mode 100644 index 0000000..0cfbc2d --- /dev/null +++ b/migrations/0002_idempotency.sql @@ -0,0 +1,10 @@ +create table idempotency_keys ( + user_id uuid not null references users(id) on delete cascade, + method text not null, + path text not null, + key text not null, + body_hash text not null, + response jsonb not null, + created_at timestamptz not null, + primary key (user_id, method, path, key) +); diff --git a/migrations/0003_attachments.sql b/migrations/0003_attachments.sql new file mode 100644 index 0000000..00d00fb --- /dev/null +++ b/migrations/0003_attachments.sql @@ -0,0 +1,22 @@ +create table attachments ( + id uuid primary key, + chat_id uuid not null references chats(id), + owner_id uuid not null references users(id), + filename text not null, + content_type text not null, + size_bytes bigint not null, + sha256 text not null, + object_key text not null unique, + created_at timestamptz not null +); + +create index attachments_chat_idx on attachments (chat_id); + +create table message_attachments ( + message_id uuid not null references messages(id), + attachment_id uuid not null references attachments(id), + position int not null, + primary key (message_id, attachment_id) +); + +create index message_attachments_attachment_idx on message_attachments (attachment_id); diff --git a/migrations/0004_jobs.sql b/migrations/0004_jobs.sql new file mode 100644 index 0000000..feddbf0 --- /dev/null +++ b/migrations/0004_jobs.sql @@ -0,0 +1,60 @@ +alter table attachments + add column status text not null default 'ready', + add column error_code text, + add column staging_key text, + add column updated_at timestamptz; + +update attachments set updated_at = created_at; + +alter table attachments + alter column status drop default, + alter column updated_at set not null, + alter column object_key drop not null; + +create table attachment_variants ( + attachment_id uuid not null references attachments(id), + pipeline_version int not null, + name text not null, + object_key text not null unique, + content_type text not null, + size_bytes bigint not null, + width int not null, + height int not null, + sha256 text not null, + primary key (attachment_id, pipeline_version, name) +); + +create table jobs ( + id uuid primary key, + attachment_id uuid not null unique references attachments(id), + pipeline_version int not null, + status text not null, + attempt int not null, + max_attempts int not null, + next_attempt_at timestamptz not null, + lease_owner text, + lease_until timestamptz, + last_error text, + created_at timestamptz not null, + updated_at timestamptz not null +); + +create index jobs_lease_idx on jobs (lease_until) where status = 'processing'; + +create table outbox ( + id bigserial primary key, + job_id uuid not null references jobs(id), + attempt int not null, + available_at timestamptz not null, + published_at timestamptz, + created_at timestamptz not null +); + +create index outbox_pending_idx on outbox (available_at) where published_at is null; + +create table job_replays ( + id bigserial primary key, + job_id uuid not null references jobs(id), + previous_error text, + replayed_at timestamptz not null +); diff --git a/migrations/0005_trace_context.sql b/migrations/0005_trace_context.sql new file mode 100644 index 0000000..ddda536 --- /dev/null +++ b/migrations/0005_trace_context.sql @@ -0,0 +1,5 @@ +alter table jobs + add column traceparent text, + add column tracestate text; + +create index jobs_queued_idx on jobs (next_attempt_at) where status = 'queued'; diff --git a/models.go b/models.go new file mode 100644 index 0000000..c6fd804 --- /dev/null +++ b/models.go @@ -0,0 +1,33 @@ +package main + +import "time" + +type User struct { + ID string + Username string + DisplayName string + PasswordHash string + CreatedAt time.Time +} + +type Chat struct { + ID string + Title string + MemberIDs []string + CreatedAt time.Time +} + +type Message struct { + ID string + ChatID string + SenderID string + Text string + CreatedAt time.Time +} + +type Session struct { + TokenHash []byte + UserID string + CreatedAt time.Time + ExpiresAt time.Time +} diff --git a/s3.go b/s3.go new file mode 100644 index 0000000..f1b381c --- /dev/null +++ b/s3.go @@ -0,0 +1,117 @@ +package main + +import ( + "context" + "io" + "net" + "net/http" + "net/url" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +type Store struct { + client *minio.Client + bucket string +} + +func openStore() (*Store, error) { + endpoint, err := url.Parse(envOr("S3_ENDPOINT", "http://s3:9000")) + if err != nil { + return nil, err + } + keyID, err := readSecret("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE") + if err != nil { + return nil, err + } + secret, err := readSecret("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE") + if err != nil { + return nil, err + } + transport := &http.Transport{ + DialContext: (&net.Dialer{Timeout: 2 * time.Second}).DialContext, + ResponseHeaderTimeout: 10 * time.Second, + MaxIdleConnsPerHost: 16, + IdleConnTimeout: 30 * time.Second, + } + client, err := minio.New(endpoint.Host, &minio.Options{ + Creds: credentials.NewStaticV4(keyID, secret, ""), + Secure: endpoint.Scheme == "https", + Region: envOr("S3_REGION", "us-east-1"), + Transport: transport, + MaxRetries: 1, + }) + if err != nil { + return nil, err + } + return &Store{client: client, bucket: envOr("S3_BUCKET", "messenger-attachments")}, nil +} + +func (s *Store) Put(ctx context.Context, key string, r io.Reader, size int64, contentType string) (err error) { + ctx, sp := childSpan(ctx, "s3 put", trace.SpanKindClient, attribute.String("s3.operation", "put")) + defer func() { endSpan(sp, err) }() + _, err = s.client.PutObject(ctx, s.bucket, key, r, size, minio.PutObjectOptions{ + ContentType: contentType, + PartSize: 5 << 20, + }) + return err +} + +func (s *Store) Get(ctx context.Context, key string) (_ *minio.Object, _ minio.ObjectInfo, err error) { + ctx, sp := childSpan(ctx, "s3 get", trace.SpanKindClient, attribute.String("s3.operation", "get")) + defer func() { endSpan(sp, err) }() + obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, minio.ObjectInfo{}, err + } + info, err := obj.Stat() + if err != nil { + obj.Close() + return nil, minio.ObjectInfo{}, err + } + return obj, info, nil +} + +func (s *Store) Remove(ctx context.Context, key string) (err error) { + ctx, sp := childSpan(ctx, "s3 remove", trace.SpanKindClient, attribute.String("s3.operation", "remove")) + defer func() { endSpan(sp, err) }() + return s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}) +} + +func (s *Store) List(ctx context.Context, prefix string) <-chan minio.ObjectInfo { + return s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) +} + +func isNoSuchKey(err error) bool { + return minio.ToErrorResponse(err).Code == "NoSuchKey" +} + +func s3Init(ctx context.Context) error { + s, err := openStore() + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + for { + exists, err := s.client.BucketExists(ctx, s.bucket) + if err == nil { + if exists { + return nil + } + err = s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: envOr("S3_REGION", "us-east-1")}) + if err == nil || minio.ToErrorResponse(err).Code == "BucketAlreadyOwnedByYou" { + return nil + } + } + select { + case <-ctx.Done(): + return err + case <-time.After(2 * time.Second): + } + } +} diff --git a/scripts/mksecrets.sh b/scripts/mksecrets.sh new file mode 100755 index 0000000..dfe9f53 --- /dev/null +++ b/scripts/mksecrets.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +mkdir -p "$ROOT/secrets" +cd "$ROOT/secrets" +gen() { + [ -f "$1" ] || openssl rand -hex "$2" | tr -d '\n' > "$1" +} +if [ ! -f postgres_password ]; then + openssl rand -base64 24 | tr -d '\n' > postgres_password +fi +PASS=$(cat postgres_password) +ENC=$(printf '%s' "$PASS" | python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.stdin.read(), safe=""))') +printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$ENC" > database_url +printf 'redis://valkey:6379/0\n' > cache_url +gen broker_password 24 +printf 'requirepass %s\n' "$(cat broker_password)" > broker.conf +printf 'redis://:%s@broker:6379/0\n' "$(cat broker_password)" > broker_url +gen grafana_admin_password 12 +gen s3_admin_access_key_id 8 +gen s3_admin_secret_access_key 20 +gen s3_access_key_id 8 +gen s3_secret_access_key 20 +BUCKET=$(grep -E '^S3_BUCKET=' "$ROOT/.env" 2>/dev/null | cut -d= -f2 || true) +BUCKET=${BUCKET:-messenger-attachments} +printf '{"identities":[{"name":"admin","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write","List","Tagging"]},{"name":"api","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Read:%s","Write:%s","List:%s"]}]}\n' \ + "$(cat s3_admin_access_key_id)" "$(cat s3_admin_secret_access_key)" "$(cat s3_access_key_id)" "$(cat s3_secret_access_key)" "$BUCKET" "$BUCKET" "$BUCKET" > s3_config.json +chmod 644 ./* +echo "secrets ready in $ROOT/secrets" diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..25b2375 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +docker compose --env-file .env restart postgres valkey broker s3 prometheus loki tempo +docker compose --env-file .env up -d --wait postgres valkey broker s3 prometheus loki tempo +docker compose --env-file .env restart otel-collector grafana api lb janitor worker diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 0000000..9cee552 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +dc="docker compose --env-file .env" +old=$($dc ps -q api) +n=$(echo "$old" | wc -w) +$dc build migrate +$dc up -d --no-deps migrate janitor worker +$dc up -d --no-deps --no-recreate --scale api=$((n * 2)) api +$dc up -d --wait --no-deps --no-recreate --scale api=$((n * 2)) api +for c in $old; do + docker stop -t 10 "$c" >/dev/null + docker rm "$c" >/dev/null +done +$dc up -d --wait --no-deps --no-recreate --scale api=$n api diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..1283ec1 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?}" +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +id=$(docker compose --env-file .env ps -q api | while read -r c; do + case "$c" in "$TARGET_INSTANCE_ID"*) echo "$c";; esac +done) +[ -n "$id" ] || exit 1 +docker stop -t 10 "$id" >/dev/null diff --git a/telemetry.go b/telemetry.go new file mode 100644 index 0000000..0bb62e8 --- /dev/null +++ b/telemetry.go @@ -0,0 +1,352 @@ +package main + +import ( + "context" + "log/slog" + "os" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/contrib/bridges/otelslog" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/propagation" + sdklog "go.opentelemetry.io/otel/sdk/log" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" +) + +const appVersion = "1.6.0" + +var ( + tracer = otel.Tracer("messenger") + meter = otel.Meter("messenger") + + httpRequests = mustCounter("http.server.requests", "{request}", "HTTP requests by method, route template and status class") + httpDuration = mustHistogram("http.server.request.duration", "s", "HTTP server request duration", + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10) + jobsProcessed = mustCounter("jobs.processed", "{job}", "Processed job attempts by outcome") + jobDuration = mustHistogram("job.processing.duration", "s", "Job processing duration", + 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60) + jobTimeToReady = mustHistogram("job.time_to_ready", "s", "Time from durable acceptance to ready", + 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300) + + exportErrors atomic.Int64 +) + +type exportErrorHandler struct{} + +func (exportErrorHandler) Handle(error) { exportErrors.Add(1) } + +func mustCounter(name, unit, desc string) metric.Int64Counter { + c, err := meter.Int64Counter(name, metric.WithUnit(unit), metric.WithDescription(desc)) + if err != nil { + panic(err) + } + return c +} + +func mustHistogram(name, unit, desc string, buckets ...float64) metric.Float64Histogram { + h, err := meter.Float64Histogram(name, metric.WithUnit(unit), metric.WithDescription(desc), + metric.WithExplicitBucketBoundaries(buckets...)) + if err != nil { + panic(err) + } + return h +} + +type fanoutHandler struct { + stdout, remote slog.Handler + service string +} + +func (h fanoutHandler) Enabled(ctx context.Context, l slog.Level) bool { + return h.stdout.Enabled(ctx, l) +} + +func (h fanoutHandler) Handle(ctx context.Context, r slog.Record) error { + local := r.Clone() + local.AddAttrs(slog.String("service.name", h.service), slog.String("event", r.Message)) + if sc := trace.SpanContextFromContext(ctx); sc.IsValid() { + local.AddAttrs(slog.String("trace_id", sc.TraceID().String()), slog.String("span_id", sc.SpanID().String())) + } + err := h.stdout.Handle(ctx, local) + remote := r.Clone() + remote.AddAttrs(slog.String("event", r.Message)) + _ = h.remote.Handle(ctx, remote) + return err +} + +func (h fanoutHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return fanoutHandler{stdout: h.stdout.WithAttrs(attrs), remote: h.remote.WithAttrs(attrs), service: h.service} +} + +func (h fanoutHandler) WithGroup(name string) slog.Handler { + return fanoutHandler{stdout: h.stdout.WithGroup(name), remote: h.remote.WithGroup(name), service: h.service} +} + +func setupTelemetry(ctx context.Context, defaultService string) (*slog.Logger, func(context.Context)) { + service := envOr("OTEL_SERVICE_NAME", defaultService) + res, err := resource.New(ctx, + resource.WithAttributes( + attribute.String("service.name", service), + attribute.String("service.version", appVersion), + attribute.String("service.instance.id", instanceID), + attribute.String("deployment.environment.name", envOr("APP_ENV", "lab")), + ), + resource.WithFromEnv(), + ) + if err != nil { + panic(err) + } + otel.SetErrorHandler(exportErrorHandler{}) + otel.SetTextMapPropagator(propagation.TraceContext{}) + + traceExp, err := otlptracehttp.New(ctx, + otlptracehttp.WithTimeout(3*time.Second), + otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: true, InitialInterval: time.Second, MaxInterval: 4 * time.Second, MaxElapsedTime: 8 * time.Second})) + if err != nil { + panic(err) + } + tp := sdktrace.NewTracerProvider( + sdktrace.WithResource(res), + sdktrace.WithBatcher(traceExp, + sdktrace.WithBatchTimeout(time.Second), + sdktrace.WithMaxQueueSize(2048), + sdktrace.WithMaxExportBatchSize(512), + sdktrace.WithExportTimeout(5*time.Second)), + ) + otel.SetTracerProvider(tp) + + metricExp, err := otlpmetrichttp.New(ctx, + otlpmetrichttp.WithTimeout(3*time.Second), + otlpmetrichttp.WithRetry(otlpmetrichttp.RetryConfig{Enabled: true, InitialInterval: time.Second, MaxInterval: 4 * time.Second, MaxElapsedTime: 8 * time.Second})) + if err != nil { + panic(err) + } + mp := sdkmetric.NewMeterProvider( + sdkmetric.WithResource(res), + sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExp, sdkmetric.WithInterval(10*time.Second), sdkmetric.WithTimeout(5*time.Second))), + ) + otel.SetMeterProvider(mp) + + logExp, err := otlploghttp.New(ctx, + otlploghttp.WithTimeout(3*time.Second), + otlploghttp.WithRetry(otlploghttp.RetryConfig{Enabled: true, InitialInterval: time.Second, MaxInterval: 4 * time.Second, MaxElapsedTime: 8 * time.Second})) + if err != nil { + panic(err) + } + lp := sdklog.NewLoggerProvider( + sdklog.WithResource(res), + sdklog.WithProcessor(sdklog.NewBatchProcessor(logExp, + sdklog.WithMaxQueueSize(2048), + sdklog.WithExportInterval(time.Second), + sdklog.WithExportTimeout(5*time.Second))), + ) + + registerProcessMetrics() + log := slog.New(fanoutHandler{ + stdout: slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}), + remote: otelslog.NewHandler("messenger", otelslog.WithLoggerProvider(lp)), + service: service, + }) + shutdown := func(ctx context.Context) { + _ = tp.Shutdown(ctx) + _ = mp.Shutdown(ctx) + _ = lp.Shutdown(ctx) + } + return log, shutdown +} + +func observableGauge(name, unit, desc string, cb func(context.Context, metric.Int64Observer) error) { + _, err := meter.Int64ObservableGauge(name, metric.WithUnit(unit), metric.WithDescription(desc), metric.WithInt64Callback(cb)) + if err != nil { + panic(err) + } +} + +func observableCounter(name, unit, desc string, cb func(context.Context, metric.Float64Observer) error) { + _, err := meter.Float64ObservableCounter(name, metric.WithUnit(unit), metric.WithDescription(desc), metric.WithFloat64Callback(cb)) + if err != nil { + panic(err) + } +} + +func registerProcessMetrics() { + observableCounter("process.cpu.time", "s", "Process CPU time", func(_ context.Context, o metric.Float64Observer) error { + var ru syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &ru); err != nil { + return nil + } + sec := func(t syscall.Timeval) float64 { return float64(t.Sec) + float64(t.Usec)/1e6 } + o.Observe(sec(ru.Utime) + sec(ru.Stime)) + return nil + }) + observableGauge("process.memory.usage", "By", "Resident set size", func(_ context.Context, o metric.Int64Observer) error { + data, err := os.ReadFile("/proc/self/status") + if err != nil { + return nil + } + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "VmRSS:") { + f := strings.Fields(line) + if len(f) >= 2 { + var kb int64 + for _, c := range f[1] { + kb = kb*10 + int64(c-'0') + } + o.Observe(kb * 1024) + } + } + } + return nil + }) + observableCounter("telemetry.export.errors", "{error}", "Failed telemetry exports", func(_ context.Context, o metric.Float64Observer) error { + o.Observe(float64(exportErrors.Load())) + return nil + }) +} + +func registerPoolMetrics(db *pgxpool.Pool) { + observableGauge("db.pool.connections", "{connection}", "DB pool connections by state", func(_ context.Context, o metric.Int64Observer) error { + s := db.Stat() + o.Observe(int64(s.AcquiredConns()), metric.WithAttributes(attribute.String("state", "used"))) + o.Observe(int64(s.IdleConns()), metric.WithAttributes(attribute.String("state", "idle"))) + return nil + }) + observableGauge("db.pool.max_connections", "{connection}", "DB pool size limit", func(_ context.Context, o metric.Int64Observer) error { + o.Observe(int64(db.Stat().MaxConns())) + return nil + }) + observableCounter("db.pool.empty_acquires", "{acquire}", "Acquires that had to wait for a connection", func(_ context.Context, o metric.Float64Observer) error { + o.Observe(float64(db.Stat().EmptyAcquireCount())) + return nil + }) + observableCounter("db.pool.acquire.wait", "s", "Total time spent acquiring connections", func(_ context.Context, o metric.Float64Observer) error { + o.Observe(db.Stat().AcquireDuration().Seconds()) + return nil + }) +} + +func registerQueueMetrics(db *pgxpool.Pool) { + var mu sync.Mutex + var depth int64 + var age float64 + refresh := func(ctx context.Context) { + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + var d int64 + var a float64 + err := db.QueryRow(ctx, ` + select count(*), + coalesce(extract(epoch from now() - min(next_attempt_at) filter (where next_attempt_at <= now())), 0) + from jobs where status = 'queued'`).Scan(&d, &a) + if err != nil { + return + } + mu.Lock() + depth, age = d, a + mu.Unlock() + } + attrs := metric.WithAttributes(attribute.String("queue", jobStream)) + observableGauge("queue.depth", "{job}", "Queued jobs", func(ctx context.Context, o metric.Int64Observer) error { + refresh(ctx) + mu.Lock() + defer mu.Unlock() + o.Observe(depth, attrs) + return nil + }) + _, err := meter.Float64ObservableGauge("queue.oldest_job_age", metric.WithUnit("s"), metric.WithDescription("Age of the oldest available queued job"), + metric.WithFloat64Callback(func(_ context.Context, o metric.Float64Observer) error { + mu.Lock() + defer mu.Unlock() + o.Observe(age, attrs) + return nil + })) + if err != nil { + panic(err) + } +} + +func registerWorkerMetrics(w *Worker, concurrency int) { + observableGauge("worker.busy", "{slot}", "Jobs being processed", func(_ context.Context, o metric.Int64Observer) error { + o.Observe(w.busy.Load()) + return nil + }) + observableGauge("worker.capacity", "{slot}", "Configured worker concurrency", func(_ context.Context, o metric.Int64Observer) error { + o.Observe(int64(concurrency)) + return nil + }) +} + +func childSpan(ctx context.Context, name string, kind trace.SpanKind, attrs ...attribute.KeyValue) (context.Context, trace.Span) { + if !trace.SpanContextFromContext(ctx).IsValid() { + return ctx, trace.SpanFromContext(ctx) + } + return tracer.Start(ctx, name, trace.WithSpanKind(kind), trace.WithAttributes(attrs...)) +} + +func endSpan(sp trace.Span, err error) { + if err != nil { + sp.RecordError(err) + sp.SetStatus(codes.Error, err.Error()) + } + sp.End() +} + +type traceCarrier struct{ traceparent, tracestate string } + +func injectTrace(ctx context.Context) traceCarrier { + m := propagation.MapCarrier{} + otel.GetTextMapPropagator().Inject(ctx, m) + return traceCarrier{traceparent: m.Get("traceparent"), tracestate: m.Get("tracestate")} +} + +func extractTrace(ctx context.Context, c traceCarrier) context.Context { + if c.traceparent == "" { + return ctx + } + m := propagation.MapCarrier{"traceparent": c.traceparent} + if c.tracestate != "" { + m["tracestate"] = c.tracestate + } + return otel.GetTextMapPropagator().Extract(ctx, m) +} + +type dbTracer struct{} + +type dbSpanKey struct{} + +func (dbTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context { + if !trace.SpanContextFromContext(ctx).IsValid() { + return ctx + } + op := "query" + if f := strings.Fields(data.SQL); len(f) > 0 { + op = strings.ToUpper(f[0]) + } + ctx, sp := tracer.Start(ctx, "db "+op, trace.WithSpanKind(trace.SpanKindClient), trace.WithAttributes( + attribute.String("db.system.name", "postgresql"), + attribute.String("db.operation.name", op), + attribute.String("db.query.text", data.SQL), + )) + return context.WithValue(ctx, dbSpanKey{}, sp) +} + +func (dbTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + if sp, ok := ctx.Value(dbSpanKey{}).(trace.Span); ok { + endSpan(sp, data.Err) + } +} diff --git a/tests/attachments.py b/tests/attachments.py new file mode 100644 index 0000000..871480c --- /dev/null +++ b/tests/attachments.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +import hashlib +import uuid +from pathlib import Path +from client import Client, arguments, check, run +from image_gen import png, png_header_only +from stack import wait_status + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() +MAX_BYTES = 10 * 1024 * 1024 + + +def upload(c, actor, chat, data, expected, content_type='image/png', filename='file.png'): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, expected, + actor=actor, raw=data, headers={'Content-Type': content_type}, + schema='Attachment' if expected == 202 else None) + + +def post_message(c, actor, chat, ids, expected, text=''): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', expected, actor=actor, + body={'text': text, 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message' if expected == 201 else None) + + +def main(): + args = arguments('Attachment validation and access checks') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'second', 'member_ids': [alice['id'], bob['id']]}, schema='Chat')['json'] + eve_chat = c.call('POST', '/api/v1/chats', 201, actor=eve, body={'title': 'eve', 'member_ids': [eve['id'], bob['id']]}, schema='Chat')['json'] + + upload(c, alice, chat, b'x' * (MAX_BYTES + 1), 413) + upload(c, alice, chat, SAMPLE + b'\0' * MAX_BYTES, 413) + upload(c, alice, chat, png_header_only(8000, 8000), 413) + upload(c, alice, chat, SAMPLE[:len(SAMPLE) // 2], 415) + upload(c, alice, chat, SAMPLE[:-1], 415) + upload(c, alice, chat, b'\x89PNG\r\n\x1a\n' + b'garbage' * 100, 415) + upload(c, alice, chat, b'', 415) + upload(c, alice, chat, b'GIF89a' + b'\0' * 64, 415) + upload(c, alice, chat, SAMPLE, 415, content_type='image/jpeg') + upload(c, alice, chat, SAMPLE, 415, content_type='text/plain') + upload(c, alice, chat, SAMPLE, 415, content_type='application/octet-stream') + + for name in ('..%2Fx.png', 'a%5Cb.png', 'a%00b.png', 'a%0Ab.png', 'a' * 256, ''): + upload(c, alice, chat, SAMPLE, 400, filename=name) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments', 400, actor=alice, raw=SAMPLE, headers={'Content-Type': 'image/png'}) + upload(c, eve, chat, SAMPLE, 404) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=a.png', 401, raw=SAMPLE, headers={'Content-Type': 'image/png'}) + + ok = upload(c, alice, chat, SAMPLE, 202)['json'] + check(ok['sha256'] == hashlib.sha256(SAMPLE).hexdigest(), 'checksum mismatch') + check(ok['filename'] == 'file.png' and ok['status'] == 'queued' and ok['variants'] == [], 'attachment fields') + unicode_name = upload(c, alice, chat, SAMPLE, 202, filename='%D1%84%D0%BE%D1%82%D0%BE.png')['json'] + check(unicode_name['filename'] == 'фото.png', 'unicode filename changed') + + post_message(c, alice, other, [ok['id']], 404) + post_message(c, alice, chat, [str(uuid.uuid4())], 404) + post_message(c, alice, chat, [ok['id'], ok['id']], 400) + post_message(c, alice, chat, ['not-a-uuid'], 400) + post_message(c, alice, chat, [str(uuid.uuid4()) for _ in range(11)], 400) + post_message(c, alice, chat, [], 400) + post_message(c, eve, chat, [ok['id']], 404) + post_message(c, eve, eve_chat, [ok['id']], 404) + + shared = post_message(c, bob, chat, [ok['id'], unicode_name['id']], 201, text='from bob')['json'] + check(shared['attachment_ids'] == [ok['id'], unicode_name['id']], 'attachment order changed') + history = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=alice, schema='MessagePage')['json']['items'] + check(any(m['id'] == shared['id'] and m['attachment_ids'] == shared['attachment_ids'] for m in history), 'history lost attachment_ids') + + for suffix in ('', '/content'): + c.call('GET', '/api/v1/attachments/' + ok['id'] + suffix, 404, actor=eve) + c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()) + suffix, 404, actor=alice) + c.call('GET', '/api/v1/attachments/bad' + suffix, 400, actor=alice) + c.call('GET', '/api/v1/attachments/' + ok['id'] + suffix, 401) + c.call('GET', '/api/v1/attachments/' + ok['id'] + '/content?variant=bogus', 400, actor=bob) + wait_status(c, bob, ok['id'], ('ready',), 60) + download = c.call('GET', '/api/v1/attachments/' + ok['id'] + '/content', actor=bob) + check(download['bytes'] == SAMPLE, 'download differs') + + big = png(1800, 1800, noise=True) + check(len(big) < MAX_BYTES, 'generated image exceeds limit') + large = upload(c, alice, chat, big, 202)['json'] + wait_status(c, bob, large['id'], ('ready',), 60) + check(c.call('GET', '/api/v1/attachments/' + large['id'] + '/content', actor=bob)['bytes'] == big, 'large download differs') + print('PASS: attachment validation, limits and access control') + + +if __name__ == '__main__': + run(main) diff --git a/tests/dashboard.py b/tests/dashboard.py new file mode 100644 index 0000000..3f2d6f9 --- /dev/null +++ b/tests/dashboard.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +import base64 +import json +import urllib.parse +import urllib.request +from pathlib import Path +from client import ROOT, arguments, check, run + +GRAFANA = 'http://127.0.0.1:3000' + + +def call(method, path, body=None): + password = (ROOT / 'secrets/grafana_admin_password').read_text().strip() + request = urllib.request.Request(GRAFANA + path, data=json.dumps(body).encode() if body is not None else None, method=method, + headers={'Content-Type': 'application/json', + 'Authorization': 'Basic ' + base64.b64encode(('admin:' + password).encode()).decode()}) + with urllib.request.urlopen(request, timeout=30) as response: + return json.loads(response.read()) + + +def main(): + arguments('Provisioned datasources and dashboard answer their queries through Grafana') + sources = {d['uid']: d['type'] for d in call('GET', '/api/datasources')} + check(sources == {'prometheus': 'prometheus', 'loki': 'loki', 'tempo': 'tempo'}, 'datasources not provisioned: %s' % sources) + found = call('GET', '/api/search?query=Messenger') + check([d['uid'] for d in found] == ['messenger-observability'], 'dashboard not provisioned') + dashboard = call('GET', '/api/dashboards/uid/messenger-observability')['dashboard'] + on_disk = json.loads((ROOT / 'infra/grafana/dashboards/messenger.json').read_text()) + check(len(dashboard['panels']) == len(on_disk['panels']), 'dashboard differs from the file in Git') + checked = 0 + for panel in dashboard['panels']: + for target in panel.get('targets', []): + if panel['datasource']['type'] == 'tempo': + found = call('GET', '/api/datasources/proxy/uid/tempo/api/search?q=' + urllib.parse.quote(target['query']) + '&limit=5') + check('traces' in found, 'panel "%s" tempo search failed' % panel['title']) + checked += 1 + continue + query = dict(target, intervalMs=15000, maxDataPoints=500) + result = call('POST', '/api/ds/query', {'queries': [query], 'from': 'now-30m', 'to': 'now'})['results'][target['refId']] + check('error' not in result and result.get('status', 200) == 200, 'panel "%s" query failed: %s' % (panel['title'], result.get('error'))) + checked += 1 + print('grafana: 3 datasources and dashboard "%s" provisioned from Git, %d panels, %d queries executed without errors' + % (dashboard['title'], len(dashboard['panels']), checked)) + print('PASS: provisioning and dashboard queries') + + +if __name__ == '__main__': + run(main) diff --git a/tests/faults.py b/tests/faults.py new file mode 100755 index 0000000..d4edb60 --- /dev/null +++ b/tests/faults.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +import json +import re +import time +import uuid +from client import Client, Failure, arguments, check, run +from stack import (SAMPLE, broker, compose, job_row, logs, now_iso, psql, restore_api, restore_worker, upload, + variant_count, wait_api, wait_log, wait_status) + + +def job_id_of(att_id): + return psql("select id from jobs where attachment_id='%s'" % att_id) + + +def worker_with(fault, **extra): + compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env=dict({'WORKER_FAULT_POINT': fault}, **extra)) + + +def wait_s3(timeout=90): + deadline = time.monotonic() + timeout + while '(healthy)' not in compose('ps', 's3', '--format', '{{.Status}}').stdout: + check(time.monotonic() < deadline, 's3 did not recover') + time.sleep(1) + + +def ops(*args): + out = compose('run', '--rm', '--no-deps', '-T', 'worker', *args).stdout + return [json.loads(line) for line in out.splitlines() if line.startswith('{')] + + +def crash_after_commit(c, alice, bob, chat): + since = now_iso() + compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': 'after_commit'}) + compose('restart', 'lb') + wait_api(c) + name = 'crash-' + uuid.uuid4().hex[:12] + '.png' + answered = None + try: + answered = upload(c, alice, chat, filename=name) + except Failure: + pass + check(answered is None, 'api answered 202 despite injected crash') + wait_log('api', since, '"point":"after_commit"', 20) + att_id = psql("select id from attachments where filename='%s'" % name) + check(att_id, 'committed attachment not found') + restore_api(c) + ready = wait_status(c, bob, att_id, ('ready', 'failed'), 60) + check(ready['status'] == 'ready' and job_row(att_id)[:2] == ('ready', 1), 'outbox did not recover the job') + published = psql("select count(*) from outbox where job_id='%s' and published_at is not null" % job_id_of(att_id)) + print('1) api crashed after COMMIT before 202/publish: attachment=%s recovered by outbox relay (published rows=%s) -> ready attempt=1' + % (att_id, published)) + + +def crash_after_store(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + since = now_iso() + worker_with('after_store', JOB_TIMEOUT_SECONDS='10') + wait_log('worker', since, '"point":"after_store"', 60) + status, attempt, _ = job_row(att['id']) + check(status == 'processing' and attempt == 1, 'job must stay processing under expired-to-be lease') + since = now_iso() + worker_with('', JOB_TIMEOUT_SECONDS='10') + wait_log('worker', since, '"lease expired","job_id":"%s"' % job_id_of(att['id']), 90) + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + status, attempt, _ = job_row(att['id']) + check(ready['status'] == 'ready' and status == 'ready' and attempt >= 2, 'job did not finish after lease expiry') + check(variant_count(att['id']) == 3 and len(ready['variants']) == 3, 'retry duplicated variants') + thumb = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0] + body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)['bytes'] + check(len(body) == thumb['size_bytes'], 'thumbnail differs from metadata') + restore_worker() + print('2) worker crashed after S3 write before COMMIT: lease expired, reaper requeued, attempt=%d ready, variants=3' % attempt) + + +def crash_before_ack(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + job_id = job_id_of(att['id']) + since = now_iso() + worker_with('before_ack') + wait_log('worker', since, '"point":"before_ack"', 60) + wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 60) + status, attempt, _ = job_row(att['id']) + check(status == 'ready' and attempt == 1 and variant_count(att['id']) == 3, 'redelivery after commit changed result') + check(broker('XPENDING', 'attachments.jobs', 'workers').splitlines()[0] == '0', 'redelivered message not acked') + check(wait_status(c, bob, att['id'], ('ready',), 10)['status'] == 'ready', 'attachment not ready') + restore_worker() + print('3) worker crashed after COMMIT before ack: message redelivered after restart, skipped as no-op, attempt=1, variants=3') + + +def retries_and_replay(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + job_id = job_id_of(att['id']) + compose('stop', 's3') + since = now_iso() + try: + compose('up', '-d', '--no-deps', 'worker') + check('(healthy)' not in compose('ps', '-a', 's3', '--format', '{{.Status}}').stdout, 's3 must stay stopped') + failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 120) + check(failed['status'] == 'failed' and failed['error_code'] == 'retry_exhausted', 'expected retry_exhausted') + text = logs('worker', since) + delays = [int(d) for d in re.findall(r'"job retry scheduled","job_id":"%s".*?"delay_ms":(\d+)' % job_id, text)] + check(len(delays) == 4, 'expected 4 scheduled retries, got %d' % len(delays)) + for n, d in enumerate(delays): + base = 1000 * 2 ** n + check(base // 2 <= d <= base, 'delay %d outside jittered window for retry %d' % (d, n + 1)) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, body={'text': '', 'attachment_ids': [att['id']]}, + headers={'Idempotency-Key': uuid.uuid4().hex}) + check(any(x['attachment_id'] == att['id'] for x in ops('failed')), 'failed registry misses job') + print('4) s3 down: transient errors retried with backoff+jitter delays_ms=%s, then failed retry_exhausted, listed in registry' % delays) + finally: + compose('start', 's3') + wait_s3() + check(ops('replay', att['id'])[0]['result'] == 'queued', 'replay did not queue job') + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 120) + check(ready['status'] == 'ready' and variant_count(att['id']) == 3, 'replayed job did not finish') + check(ops('replay', att['id'])[0]['result'] == 'skipped_ready', 'replay of ready job must be a no-op') + check(psql("select count(*) from job_replays where job_id='%s'" % job_id) == '1', 'replay audit missing') + check(variant_count(att['id']) == 3, 'second replay created new effects') + print(' replay: failed -> queued -> ready (audit rows=1); replay of ready job skipped, variants=3') + + +def permanent_error(c, alice, bob, chat): + data = bytearray(SAMPLE) + mid = len(data) // 2 + data[mid:mid + 64] = b'\x00' * 64 + att = upload(c, alice, chat, bytes(data))['json'] + failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 60) + status, attempt, _ = job_row(att['id']) + check(failed['status'] == 'failed' and failed['error_code'] == 'invalid_image' and attempt == 1, 'corrupt image must fail without retries') + check(failed['variants'] == [], 'failed attachment has variants') + print('5) corrupted IDAT (passes header/tail check in API): failed invalid_image on attempt 1, no retries') + + +def main(): + args = arguments('Fault injection: COMMIT/publish gap, crash before COMMIT and before ack, retries, DLQ replay') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + try: + crash_after_commit(c, alice, bob, chat) + crash_after_store(c, alice, bob, chat) + crash_before_ack(c, alice, bob, chat) + retries_and_replay(c, alice, bob, chat) + permanent_error(c, alice, bob, chat) + finally: + compose('up', '-d', '--no-deps', 's3') + restore_worker() + if 'after_commit' in compose('exec', '-T', 'api', 'printenv', 'FAULT_POINT', check_rc=False).stdout: + restore_api(c) + print('PASS: outbox recovery, lease recovery, redelivery after commit, backoff with jitter, retry_exhausted, replay, permanent error') + + +if __name__ == '__main__': + run(main) diff --git a/tests/idempotency.py b/tests/idempotency.py new file mode 100644 index 0000000..951d438 --- /dev/null +++ b/tests/idempotency.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import threading +import uuid +from client import Client, Failure, arguments, check, run + + +def main(): + args = arguments('Concurrent Idempotency-Key check through the load balancer') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/messages' + parallel = 32 + key = 'idem-' + uuid.uuid4().hex + barrier = threading.Barrier(parallel) + + def send(_): + local = Client(args.base_url, args.ca_file) + barrier.wait() + return local.call('POST', path, 201, actor=alice, body={'text': 'same'}, headers={'Idempotency-Key': key}, schema='Message') + + with ThreadPoolExecutor(max_workers=parallel) as pool: + results = list(pool.map(send, range(parallel))) + ids = {r['json']['id'] for r in results} + instances = {r['headers']['X-Instance-Id'] for r in results} + check(len(ids) == 1, 'parallel requests produced %d messages' % len(ids)) + check(len(instances) >= 2, 'requests were served by one instance only') + check(len({r['json']['created_at'] for r in results}) == 1, 'created_at differs between replays') + + listing = c.call('GET', path, actor=alice, schema='MessagePage')['json']['items'] + check(sum(1 for m in listing if m['text'] == 'same') == 1, 'duplicate message in history') + + c.call('POST', path, 409, actor=alice, body={'text': 'other'}, headers={'Idempotency-Key': key}) + c.call('POST', path, 400, actor=alice, body={'text': 'x'}) + c.call('POST', path, 400, actor=alice, body={'text': 'x'}, headers={'Idempotency-Key': 'bad key!'}) + other = c.call('POST', path, 201, actor=bob, body={'text': 'same'}, headers={'Idempotency-Key': key}, schema='Message')['json'] + check(other['id'] not in ids, 'key must be scoped per user') + print('PASS: %d parallel requests, 1 message, %d instances' % (parallel, len(instances))) + + +if __name__ == '__main__': + run(main) diff --git a/tests/image_gen.py b/tests/image_gen.py new file mode 100644 index 0000000..aeb7de0 --- /dev/null +++ b/tests/image_gen.py @@ -0,0 +1,23 @@ +import os +import struct +import zlib + + +def chunk(kind, data): + body = kind + data + return struct.pack('>I', len(data)) + body + struct.pack('>I', zlib.crc32(body) & 0xffffffff) + + +def png(width, height, noise=False): + header = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + raw = bytearray() + row = width * 3 + 1 + for _ in range(height): + raw += b'\x00' + (os.urandom(row - 1) if noise else bytes(row - 1)) + data = zlib.compress(bytes(raw), 0 if noise else 6) + return header + chunk(b'IDAT', data) + chunk(b'IEND', b'') + + +def png_header_only(width, height): + header = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + return header + chunk(b'IDAT', zlib.compress(b'\x00')) + chunk(b'IEND', b'') diff --git a/tests/incidents.py b/tests/incidents.py new file mode 100644 index 0000000..835c7ff --- /dev/null +++ b/tests/incidents.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +import subprocess +import threading +import time +import uuid +from client import Client, Failure, arguments, check, run +from obs import alerts, loki, prom, prom_value, trace_search, wait_alert, wait_trace_spans +from stack import compose, restore_worker, upload, wait_status + + +class Load: + def __init__(self, c, actor, chat, uploads=False, workers=3): + self.c, self.actor, self.chat, self.uploads = c, actor, chat, uploads + self.stop = threading.Event() + self.codes = {} + self.lock = threading.Lock() + self.threads = [threading.Thread(target=self.loop, args=(i,)) for i in range(workers)] + + def record(self, code): + with self.lock: + self.codes[code] = self.codes.get(code, 0) + 1 + + def loop(self, index): + while not self.stop.is_set(): + try: + if self.uploads and index == 0: + self.c.call('POST', '/api/v1/chats/%s/attachments?filename=l-%s.png' % (self.chat['id'], uuid.uuid4().hex[:8]), + [202, 503], actor=self.actor, raw=open('tests/fixtures/sample.png', 'rb').read(), + headers={'Content-Type': 'image/png'}) + self.record('upload') + time.sleep(1) + continue + r = self.c.call('POST', '/api/v1/chats/%s/messages' % self.chat['id'], [201, 503], actor=self.actor, + body={'text': 'load ' + uuid.uuid4().hex}, headers={'Idempotency-Key': uuid.uuid4().hex}) + self.record(r['status']) + except Failure as exc: + self.record('failure') + time.sleep(0.2) + + def __enter__(self): + for t in self.threads: + t.start() + return self + + def __exit__(self, *exc): + self.stop.set() + for t in self.threads: + t.join() + + +def rss_mib(container): + out = subprocess.run(['docker', 'stats', '--no-stream', '--format', '{{.MemUsage}}', container], capture_output=True, text=True).stdout + value = out.split('/')[0].strip() + number = float(''.join(ch for ch in value if ch.isdigit() or ch == '.')) + return number * (1024 if value.endswith('GiB') else 1 if value.endswith('MiB') else 1 / 1024) + + +def worker_stopped(c, alice, bob, chat): + check(not alerts().get('QueueLagGrowing'), 'alert must start resolved') + compose('stop', 'worker') + stopped = time.time() + with Load(c, alice, chat, uploads=True, workers=1): + fired_after = wait_alert('QueueLagGrowing', True, 180) + age = prom_value('max(queue_oldest_job_age_seconds)') + depth = prom_value('max(queue_depth)') + rate = prom_value('sum(rate(jobs_processed_total[1m]))', 0) + busy = prom_value('sum(worker_busy)') + stuck = trace_search('{ resource.service.name = "messenger-api" && name = "attachments.jobs publish" }', 5, stopped)[0] + spans = wait_trace_spans(stuck['traceID'], lambda s: any(x['name'] == 'attachments.jobs publish' for x in s)) + consumers = [s for s in spans if s['kind'] == 'SPAN_KIND_CONSUMER'] + silent = loki('{service_name="messenger-worker"} | event="job ready"', 1, 5) + check(not consumers, 'stuck job already has a consumer span') + check(age is not None and age > 20 and depth and depth > 0, 'queue lag not visible in metrics') + check(rate < 0.01, 'jobs still processed while worker is stopped') + restart = time.monotonic() + restore_worker() + resolved_after = wait_alert('QueueLagGrowing', False, 180) + deadline = time.monotonic() + 120 + while prom_value('max(queue_depth)', 0) > 0 and time.monotonic() < deadline: + time.sleep(2) + drained = time.monotonic() - restart + check(prom_value('max(queue_depth)', 0) == 0, 'backlog did not drain') + print('1) worker stopped: QueueLagGrowing firing after %.0fs; oldest job age=%.0fs depth=%d jobs/s=%.2f worker_busy=%s' + % (fired_after, age, depth, rate, busy)) + print(' diagnosis from stored data: trace %s has the api server and "attachments.jobs publish" spans and no consumer span; ' + 'no "job ready" worker log in Loki during the last minute (%d lines)' % (stuck['traceID'], len(silent))) + print(' restored: worker started, alert resolved after %.0fs, backlog drained in %.0fs' % (resolved_after, drained)) + + +def slow_database(c, alice, bob, chat): + check(not alerts().get('ApiHighErrorRate'), 'alert must start resolved') + lock = subprocess.Popen(['docker', 'compose', '--env-file', '.env', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', + '-c', 'begin; lock table messages in access exclusive mode; select pg_sleep(45); commit;'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + time.sleep(2) + with Load(c, alice, chat, workers=4) as load: + fired_after = wait_alert('ApiHighErrorRate', True, 120) + ratio = prom_value('sum(rate(http_server_requests_total{http_response_status_class="5xx"}[1m])) / sum(rate(http_server_requests_total[1m]))') + by_route = {r['metric']['http_route']: float(r['value'][1]) for r in + prom('sum by (http_route) (rate(http_server_requests_total{http_response_status_class="5xx"}[1m]))') if float(r['value'][1]) > 0} + p99 = prom_value('histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket[1m])))') + used = prom_value('sum(db_pool_connections{state="used",job="messenger-api"})') + wait = prom_value('sum(rate(db_pool_acquire_wait_seconds_total{job="messenger-api"}[1m]))', 0) + errors = trace_search('{ resource.service.name = "messenger-api" && status = error && name =~ "POST .*messages" }', 5) + check(errors, 'no error trace found') + spans = wait_trace_spans(errors[0]['traceID'], lambda s: len(s) > 1) + server = [s for s in spans if s['kind'] == 'SPAN_KIND_SERVER'][0] + slow_db = max((s for s in spans if s['name'].startswith('db ')), key=lambda s: s['duration_us']) + share = slow_db['duration_us'] / server['duration_us'] + check(ratio and ratio > 0.05, 'error ratio not visible') + check(share > 0.8 and slow_db['status'] == 'STATUS_CODE_ERROR', 'time is not spent in the database span') + check(set(by_route) <= {'/api/v1/chats/{chat_id}/messages'}, 'unexpected failing routes: %s' % by_route) + lock.wait(timeout=90) + resolved_after = wait_alert('ApiHighErrorRate', False, 180) + print('2) database lock on messages: ApiHighErrorRate firing after %.0fs; 5xx ratio=%.2f only on %s; p99=%.1fs; pool used=%s acquire wait=%.2f s/s' + % (fired_after, ratio, sorted(by_route), p99 or 0, used, wait)) + print(' diagnosis from stored data: error trace %s, server span %d ms, child "%s" %d ms (%.0f%%) with status error, ' + 'handler has no other slow child; requests: %s' % (errors[0]['traceID'], server['duration_us'] // 1000, slow_db['name'], + slow_db['duration_us'] // 1000, share * 100, load.codes)) + print(' restored: lock released, alert resolved after %.0fs' % resolved_after) + + +def storage_down(c, alice, bob, chat): + stores = ['prometheus', 'loki', 'tempo'] + container = compose('ps', '-q', 'otel-collector').stdout.strip() + before = {name: prom_value('sum(%s)' % name, 0) for name in + ('otelcol_exporter_send_failed_spans', 'otelcol_exporter_send_failed_log_records', 'otelcol_exporter_send_failed_metric_points')} + peak = rss_mib(container) + compose('stop', *stores) + samples = [] + attachments = [] + try: + with Load(c, alice, chat, uploads=True, workers=3) as load: + for _ in range(20): + time.sleep(4) + samples.append(rss_mib(container)) + codes = dict(load.codes) + for _ in range(3): + attachments.append(upload(c, alice, chat)['json']['id']) + ready = [wait_status(c, bob, a, ('ready', 'failed'), 60)['status'] for a in attachments] + msg = c.message(alice, chat, 'during outage') + state = subprocess.run(['docker', 'inspect', '-f', '{{.RestartCount}} {{.State.OOMKilled}}', container], capture_output=True, text=True).stdout.split() + finally: + compose('up', '-d', '--wait', *stores) + check(codes.get(201, 0) > 20 and 'failure' not in codes and 503 not in codes, 'API was affected by telemetry outage: %s' % codes) + check(ready == ['ready'] * 3 and msg['status'] == 201, 'messages or jobs affected by telemetry outage') + check(state == ['0', 'false'], 'collector restarted or was OOM killed') + check(max(samples) < 230, 'collector memory unbounded: %.0f MiB' % max(samples)) + time.sleep(30) + after = {name: prom_value('sum(%s)' % name, 0) for name in before} + check(after['otelcol_exporter_send_failed_spans'] > before['otelcol_exporter_send_failed_spans'], 'collector did not count dropped spans') + exports = prom_value('sum(telemetry_export_errors_total)', 0) + fresh = loki('{service_name="messenger-api"} | event="request"', 2, 5) + check(fresh, 'logs did not resume after storage recovery') + print('3) prometheus, loki and tempo stopped for ~80s under load: requests=%s, 3/3 uploads ready, message 201, no 5xx' % codes) + print(' collector rss MiB: start=%.0f peak=%.0f during outage (limit 256, memory_limiter 160), restarts/oom=%s' % (peak, max(samples), ' '.join(state))) + print(' telemetry lost and counted: collector send_failed spans +%d, log records +%d, metric points +%d; sdk export errors=%d' + % (after['otelcol_exporter_send_failed_spans'] - before['otelcol_exporter_send_failed_spans'], + after['otelcol_exporter_send_failed_log_records'] - before['otelcol_exporter_send_failed_log_records'], + after['otelcol_exporter_send_failed_metric_points'] - before['otelcol_exporter_send_failed_metric_points'], exports)) + print(' after restore new logs and metrics are stored again; data from the outage window is lost, data before it remains') + + +def main(): + def configure(p): + p.add_argument('--only', choices=['worker', 'database', 'storage']) + args = arguments('Controlled incidents with alerts and diagnosis from stored telemetry', configure) + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + try: + if args.only in (None, 'worker'): + worker_stopped(c, alice, bob, chat) + if args.only in (None, 'database'): + slow_database(c, alice, bob, chat) + if args.only in (None, 'storage'): + storage_down(c, alice, bob, chat) + finally: + restore_worker() + compose('up', '-d', '--no-deps', 'prometheus', 'loki', 'tempo', check_rc=False) + print('PASS: ' + {'worker': 'stopped worker detected by alert and diagnosed from telemetry', + 'database': 'slow database detected by alert and diagnosed from telemetry', + 'storage': 'telemetry storage outage does not affect the API and loss is bounded and counted', + None: 'two alerting incidents diagnosed from telemetry, storage outage does not affect the API'}[args.only]) + + +if __name__ == '__main__': + run(main) diff --git a/tests/obs.py b/tests/obs.py new file mode 100644 index 0000000..daf6ae6 --- /dev/null +++ b/tests/obs.py @@ -0,0 +1,125 @@ +import json +import time +import urllib.parse +import urllib.request +from client import Failure, check + +PROM = 'http://127.0.0.1:9090' +LOKI = 'http://127.0.0.1:3100' +TEMPO = 'http://127.0.0.1:3200' + + +def get(url, params=None, timeout=10): + if params: + url += '?' + urllib.parse.urlencode(params) + with urllib.request.urlopen(url, timeout=timeout) as response: + return json.loads(response.read()) + + +def prom(expr): + data = get(PROM + '/api/v1/query', {'query': expr}) + check(data['status'] == 'success', 'prometheus query failed: ' + expr) + return data['data']['result'] + + +def prom_value(expr, default=None): + result = prom(expr) + if not result: + return default + value = float(result[0]['value'][1]) + return default if value != value else value + + +def series(name): + return get(PROM + '/api/v1/series', {'match[]': name})['data'] + + +def alerts(): + return {a['labels']['alertname']: a['state'] for a in get(PROM + '/api/v1/alerts')['data']['alerts']} + + +def wait_alert(name, firing, timeout=120): + deadline = time.monotonic() + timeout + started = time.monotonic() + while time.monotonic() < deadline: + state = alerts().get(name) + if (state == 'firing') == firing: + return time.monotonic() - started + time.sleep(2) + raise Failure('alert %s did not become %s' % (name, 'firing' if firing else 'resolved')) + + +def loki(query, minutes=15, limit=200): + end = time.time() + data = get(LOKI + '/loki/api/v1/query_range', { + 'query': query, 'limit': limit, 'direction': 'backward', + 'start': int((end - minutes * 60) * 1e9), 'end': int((end + 60) * 1e9)}) + rows = [] + for stream in data['data']['result']: + for ts, line in stream['values']: + rows.append((int(ts), stream['stream'], line)) + return sorted(rows, key=lambda r: r[0]) + + +def wait_loki(query, timeout=40, **kw): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + rows = loki(query, **kw) + if rows: + return rows + time.sleep(2) + raise Failure('loki returned nothing for ' + query) + + +def trace(trace_id, timeout=40): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + return flatten(get(TEMPO + '/api/traces/' + trace_id)) + except Exception: + time.sleep(2) + raise Failure('tempo has no trace ' + trace_id) + + +def attrs(items): + out = {} + for a in items: + v = a['value'] + out[a['key']] = v.get('stringValue', v.get('intValue', v.get('boolValue', v.get('doubleValue')))) + return out + + +def flatten(data): + spans = [] + for batch in data.get('batches', []): + service = attrs(batch['resource']['attributes']).get('service.name') + for scope in batch['scopeSpans']: + for s in scope['spans']: + spans.append({ + 'service': service, 'name': s['name'], 'kind': s['kind'], 'id': s['spanId'], + 'parent': s.get('parentSpanId', ''), 'start': int(s['startTimeUnixNano']), + 'duration_us': (int(s['endTimeUnixNano']) - int(s['startTimeUnixNano'])) // 1000, + 'attributes': attrs(s.get('attributes', [])), 'status': s.get('status', {}).get('code', ''), + }) + return sorted(spans, key=lambda s: s['start']) + + +def trace_search(query, limit=20, since=None): + params = {'q': query, 'limit': limit} + if since: + params.update(start=int(since), end=int(time.time()) + 60) + return get(TEMPO + '/api/search', params).get('traces', []) + + +def wait_trace_spans(trace_id, predicate, timeout=60): + deadline = time.monotonic() + timeout + spans = [] + while time.monotonic() < deadline: + try: + spans = trace(trace_id, 5) + except Failure: + spans = [] + if spans and predicate(spans): + return spans + time.sleep(2) + raise Failure('trace %s did not reach the expected shape' % trace_id) diff --git a/tests/observability.py b/tests/observability.py new file mode 100644 index 0000000..2cacad0 --- /dev/null +++ b/tests/observability.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +import json +import re +import time +import uuid +from client import Client, arguments, check, run +from faults import ops, wait_s3 +from obs import loki, prom, series, wait_loki, wait_trace_spans +from stack import compose, restore_worker, upload, wait_status + +SIGNALS = [ + 'http_server_requests_total', 'http_server_request_duration_seconds_bucket', 'jobs_processed_total', + 'job_processing_duration_seconds_bucket', 'job_time_to_ready_seconds_bucket', 'queue_depth', + 'queue_oldest_job_age_seconds', 'db_pool_connections', 'db_pool_max_connections', 'worker_busy', + 'worker_capacity', 'process_cpu_time_seconds_total', 'process_memory_usage_bytes', +] +FORBIDDEN = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'filename', 'url', 'http_url', 'path', 'text'} +UUID = re.compile(r'[0-9a-f]{8}-[0-9a-f]{4}-') + + +def signals_and_labels(): + for name in SIGNALS: + check(prom('count(%s)' % name), 'metric %s has no samples' % name) + seen = set() + for name in SIGNALS: + for s in series(name): + seen |= set(s) - {'__name__'} + check(not UUID.search(s.get('http_route', '')), 'raw identifier in route label') + check(not (seen & FORBIDDEN), 'high-cardinality labels present: %s' % sorted(seen & FORBIDDEN)) + routes = {s['http_route'] for s in series('http_server_requests_total')} + check('/api/v1/chats/{chat_id}/messages' in routes, 'route template missing') + print('metrics: %d signals present, labels=%s, routes are templates (%d)' % (len(SIGNALS), sorted(seen), len(routes))) + + +def chain(c, alice, bob, chat): + resp = upload(c, alice, chat) + request_id = resp['headers']['X-Request-Id'] + att = resp['json'] + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + check(ready['status'] == 'ready', 'attachment did not become ready') + rows = wait_loki('{service_name="messenger-api"} | event="request" | request_id="%s"' % request_id) + trace_id = rows[-1][1].get('trace_id') + check(trace_id and re.fullmatch(r'[0-9a-f]{32}', trace_id), 'request log has no trace_id') + check(rows[-1][1].get('span_id'), 'request log has no span_id') + spans = wait_trace_spans(trace_id, lambda s: any(x['kind'] == 'SPAN_KIND_CONSUMER' for x in s) + and any(x['name'] == 'db COMMIT' and x['service'] == 'messenger-worker' for x in s) + and any(x['name'] == 's3 remove' for x in s)) + by_id = {s['id']: s for s in spans} + server = [s for s in spans if s['kind'] == 'SPAN_KIND_SERVER'][0] + producer = [s for s in spans if s['kind'] == 'SPAN_KIND_PRODUCER'][0] + consumer = [s for s in spans if s['kind'] == 'SPAN_KIND_CONSUMER'][0] + check(server['service'] == 'messenger-api' and producer['service'] == 'messenger-api', 'producer must belong to api') + check(consumer['service'] == 'messenger-worker', 'consumer must belong to worker') + check(producer['parent'] == server['id'], 'producer span is not a child of the request span') + check(consumer['parent'] == producer['id'], 'consumer span is not a child of the producer span') + check(consumer['attributes'].get('attachment.id') == att['id'], 'consumer span misses attachment.id') + names = {(s['service'], s['name']) for s in spans} + for want in [('messenger-api', 's3 put'), ('messenger-api', 'db INSERT'), ('messenger-worker', 's3 get'), + ('messenger-worker', 'image render'), ('messenger-worker', 's3 put'), ('messenger-worker', 'db COMMIT'), + ('messenger-worker', 's3 remove')]: + check(want in names, 'span %s missing' % (want,)) + check(all(s['parent'] in by_id or not s['parent'] for s in spans), 'orphan spans in trace') + worker_logs = wait_loki('{service_name="messenger-worker"} | event="job ready" | trace_id="%s"' % trace_id) + check(worker_logs[-1][1].get('span_id'), 'worker log has no span_id') + print('chain: request_id=%s trace_id=%s spans=%d api->producer->consumer->s3/db linked, api and worker logs carry trace_id' + % (request_id, trace_id, len(spans))) + return att, trace_id + + +def secrets(c, alice, chat, trace_id): + marker = 'secret-body-' + uuid.uuid4().hex + resp = c.message(alice, chat, marker) + request_id = resp['headers']['X-Request-Id'] + time.sleep(8) + rows = wait_loki('{service_name="messenger-api"} | event="request" | request_id="%s"' % request_id) + message_trace = rows[-1][1].get('trace_id') + raw = json.dumps(wait_trace_spans(message_trace, lambda s: len(s) > 1)) + json.dumps(wait_trace_spans(trace_id, lambda s: True)) + for needle, what in ((marker, 'message body'), (alice['token'], 'session token')): + check(needle not in raw, '%s leaked into traces' % what) + check(not loki('{service_name=~"messenger-.*"} |= "%s"' % needle), '%s leaked into logs' % what) + check(not loki('{service_name=~"messenger-.*"} |~ "(?i)authorization|bearer "'), 'authorization material in logs') + print('secrets: message body and session token absent from Loki and Tempo') + + +def retry_trace(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + compose('stop', 's3') + try: + compose('up', '-d', '--no-deps', 'worker') + failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 120) + check(failed['status'] == 'failed', 'expected retry_exhausted while S3 is down') + finally: + compose('start', 's3') + wait_s3() + check(ops('replay', att['id'])[0]['result'] == 'queued', 'replay did not queue job') + check(wait_status(c, bob, att['id'], ('ready', 'failed'), 120)['status'] == 'ready', 'replay did not finish') + rows = wait_loki('{service_name="messenger-worker"} | event="job ready" | attachment_id="%s"' % att['id']) + trace_id = rows[-1][1]['trace_id'] + spans = wait_trace_spans(trace_id, lambda s: sum(x['kind'] == 'SPAN_KIND_CONSUMER' for x in s) >= 6 + and any(x['name'] == 'db COMMIT' for x in s)) + consumers = [s for s in spans if s['kind'] == 'SPAN_KIND_CONSUMER'] + attempts = sorted(int(s['attributes']['job.attempt']) for s in consumers) + check(attempts == list(range(1, len(attempts) + 1)), 'attempts are not sequential: %s' % attempts) + errors = [s for s in consumers if s['status'] == 'STATUS_CODE_ERROR'] + check(len(errors) == 5, 'expected 5 failed attempts in the trace, got %d' % len(errors)) + check(sum(1 for s in spans if s['kind'] == 'SPAN_KIND_PRODUCER') == len(consumers), 'each attempt needs its publish span') + print('retry: one trace %s holds %d attempts (%d failed with S3 down, attempt %d ready after replay)' + % (trace_id, len(consumers), len(errors), attempts[-1])) + + +def main(): + args = arguments('Observability signals, trace propagation, correlation and redaction') + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + try: + att, trace_id = chain(c, alice, bob, chat) + signals_and_labels() + secrets(c, alice, chat, trace_id) + retry_trace(c, alice, bob, chat) + finally: + compose('start', 's3', check_rc=False) + restore_worker() + print('PASS: signals, producer/consumer trace chain, log correlation, retry in trace, no secrets in telemetry') + + +if __name__ == '__main__': + run(main) diff --git a/tests/queue_recovery.py b/tests/queue_recovery.py new file mode 100755 index 0000000..0c420d8 --- /dev/null +++ b/tests/queue_recovery.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +import json +import time +from client import Client, arguments, check, run +from image_probe import jpeg_size +from stack import broker, compose, job_row, logs, now_iso, psql, upload, variant_count, wait_api, wait_log, wait_status + + +def stream_has(job_id): + return job_id in broker('XRANGE', 'attachments.jobs', '-', '+') + + +def main(): + args = arguments('Worker stop, API/broker restart and redelivery scenario') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + + compose('stop', 'worker') + try: + att = upload(c, alice, chat)['json'] + check(att['status'] == 'queued' and att['variants'] == [], 'upload must return queued snapshot') + msg = c.message(alice, chat, '', attachments=[att['id']])['json'] + check(msg['attachment_ids'] == [att['id']], 'message lost queued attachment') + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 409, actor=bob) + c.message(bob, chat, 'text works while worker is down') + job_id = psql("select id from jobs where attachment_id='%s'" % att['id']) + deadline = time.monotonic() + 10 + while not stream_has(job_id): + check(time.monotonic() < deadline, 'job was not published to broker') + time.sleep(0.5) + check(psql("select count(*) from outbox where job_id='%s' and published_at is null" % job_id) == '0', 'outbox not marked') + print('accepted: attachment=%s job=%s status=queued, job in stream, worker stopped' % (att['id'], job_id)) + + compose('restart', 'broker', 'api') + compose('up', '-d', '--wait', '--no-deps', 'broker', 'api') + compose('restart', 'lb') + wait_api(c) + check(stream_has(job_id), 'job lost after broker restart') + check(c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status'] == 'queued', 'status changed without worker') + print('restarted broker and api: job still in stream, attachment still queued') + finally: + compose('start', 'worker') + + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + check(ready['status'] == 'ready', 'job did not finish after worker start') + thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob) + check(jpeg_size(thumb['bytes']) == (256, 160), 'thumbnail size') + check(job_row(att['id'])[:2] == ('ready', 1) and variant_count(att['id']) == 3, 'unexpected job state or variants') + print('worker started: attachment ready, attempt=1, variants=3') + + since = now_iso() + before = psql("select updated_at from attachments where id='%s'" % att['id']) + payload = json.dumps({'schema_version': 1, 'job_id': job_id, 'type': 'attachment.process.v1', 'attachment_id': att['id'], + 'pipeline_version': 1, 'attempt': 1, 'created_at': ready['created_at']}) + broker('XADD', 'attachments.jobs', '*', 'job', payload) + wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 30) + time.sleep(1) + check(psql("select updated_at from attachments where id='%s'" % att['id']) == before, 'duplicate delivery changed attachment') + check(variant_count(att['id']) == 3 and job_row(att['id'])[:2] == ('ready', 1), 'duplicate delivery created new effects') + pending = broker('XPENDING', 'attachments.jobs', 'workers') + check(pending.splitlines()[0] == '0', 'duplicate message was not acked') + print('duplicate delivery: skipped as no-op, acked, variants=3, updated_at unchanged') + print('PASS: worker stop, broker/api restart, redelivery idempotency') + + +if __name__ == '__main__': + run(main) diff --git a/tests/s3_checks.py b/tests/s3_checks.py new file mode 100644 index 0000000..2371df2 --- /dev/null +++ b/tests/s3_checks.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +import hashlib +import subprocess +import time +import uuid +from pathlib import Path +from client import Client, Failure, arguments, check, run +from stack import wait_status + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() +BUCKET = 'messenger-attachments' + + +def compose(*args, check_rc=True, stdin=None): + return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=check_rc, + capture_output=True, text=True, input=stdin) + + +def secret(name): + return (ROOT / 'secrets' / name).read_text().strip() + + +def s3_status(method, path, user=None, data=None): + cmd = ['exec', '-T', 's3', 'curl', '-s', '-o', '/dev/null', '-w', '%{http_code}', '-X', method] + if user: + cmd += ['--aws-sigv4', 'aws:amz:us-east-1:s3', '--user', user[0] + ':' + user[1]] + if data is not None: + cmd += ['--data-binary', data] + cmd.append('http://127.0.0.1:9000' + path) + return compose(*cmd).stdout.strip() + + +def upload(c, actor, chat, expected): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=a.png', expected, actor=actor, + raw=SAMPLE, headers={'Content-Type': 'image/png'}, schema='Attachment' if expected == 202 else None) + + +def wait_api(c): + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + try: + c.call('GET', '/health/ready') + return + except Failure: + time.sleep(1) + raise Failure('api did not recover') + + +def wait_s3(timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if compose('ps', 's3', '--format', '{{.Status}}').stdout.count('(healthy)'): + return + time.sleep(1) + raise Failure('s3 did not recover') + + +def object_key(attachment_id): + out = compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', + "select object_key from attachments where id='%s'" % attachment_id).stdout.strip() + check(out.startswith('attachments/'), 'object key not found') + return out + + +def main(): + args = arguments('S3 privacy, outage and orphan cleanup checks') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + att = upload(c, alice, chat, 202)['json'] + wait_status(c, bob, att['id'], ('ready',), 60) + key = object_key(att['id']) + check(key != 'attachments/' + att['filename'] and att['filename'] not in key, 'object key derived from filename') + + api = (secret('s3_access_key_id'), secret('s3_secret_access_key')) + admin = (secret('s3_admin_access_key_id'), secret('s3_admin_secret_access_key')) + path = '/' + BUCKET + '/' + key + check(s3_status('GET', path) == '403', 'anonymous GET must be denied') + check(s3_status('GET', '/' + BUCKET + '/') == '403', 'anonymous listing must be denied') + check(s3_status('PUT', path + '-anon', data='x') == '403', 'anonymous PUT must be denied') + check(s3_status('GET', path, ('wrong', 'wrong')) == '403', 'wrong credentials must be denied') + check(s3_status('GET', path, api) == '200', 'api credentials cannot read') + check(s3_status('PUT', '/other-bucket-' + uuid.uuid4().hex[:8], api) == '403', 'api credentials can create buckets') + ports = compose('ps', '--format', '{{.Service}} {{.Ports}}').stdout + lines = [line for line in ports.splitlines() if '->' in line] + published = [line.split()[0] for line in lines] + check(sorted(published) == ['grafana', 'lb', 'loki', 'prometheus', 'tempo'], 'unexpected published services: %s' % published) + check(all(part.startswith('127.0.0.1:') for line in lines for part in line.split(None, 1)[1].split(', ') if '->' in part), + 'published ports must be bound to 127.0.0.1') + + compose('stop', 's3') + try: + upload(c, alice, chat, 503) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob) + c.call('GET', '/api/v1/attachments/' + att['id'], 200, actor=bob, schema='Attachment') + c.message(alice, chat, 'text works while s3 is down') + finally: + compose('start', 's3') + wait_s3() + wait_api(c) + deadline = time.monotonic() + 60 + while True: + try: + body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] + check(body == SAMPLE, 'content differs after s3 restart') + break + except Failure: + check(time.monotonic() < deadline, 'download did not recover') + time.sleep(1) + upload(c, alice, chat, 202) + + orphan = '/' + BUCKET + '/attachments/' + uuid.uuid4().hex + check(s3_status('PUT', orphan, admin, 'orphan') == '200', 'cannot create orphan') + compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=3600', 'janitor', 'cleanup') + check(s3_status('GET', orphan, admin) == '200', 'fresh object was removed inside grace period') + time.sleep(1) + compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=0', 'janitor', 'cleanup') + check(s3_status('GET', orphan, admin) == '404', 'orphan was not removed') + check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == SAMPLE, 'referenced object was removed') + + check(s3_status('GET', '/' + BUCKET + '/staging/' + att['id'], admin) == '404', 'staging kept after ready') + stale = '/' + BUCKET + '/staging/' + str(uuid.uuid4()) + check(s3_status('PUT', stale, admin, 'stale') == '200', 'cannot create stale staging object') + compose('stop', 'worker') + try: + queued = upload(c, alice, chat, 202)['json'] + compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=0', 'janitor', 'cleanup') + check(s3_status('GET', '/' + BUCKET + '/staging/' + queued['id'], admin) == '200', 'staging of queued job was removed') + check(s3_status('GET', stale, admin) == '404', 'unreferenced staging object was not removed') + finally: + compose('up', '-d', '--no-deps', 'worker') + check(wait_status(c, bob, queued['id'], ('ready', 'failed'), 60)['status'] == 'ready', 'queued job lost staging') + print('PASS: private bucket, limited credentials, s3 outage, orphan and staging cleanup') + + +if __name__ == '__main__': + run(main) diff --git a/tests/scenario.py b/tests/scenario.py new file mode 100644 index 0000000..f7ddd70 --- /dev/null +++ b/tests/scenario.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import json +import subprocess +import threading +import time +import uuid +from pathlib import Path +from client import Client, Failure, arguments, check, run + +ROOT = Path(__file__).resolve().parent.parent + + +def sh(*cmd): + return subprocess.run(cmd, cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip() + + +def api_containers(): + out = sh('docker', 'compose', '--env-file', '.env', 'ps', '-q', 'api') + return [x[:12] for x in out.split()] + + +def wait_healthy(count, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + out = sh('docker', 'compose', '--env-file', '.env', 'ps', 'api', '--format', '{{.Status}}') + if out.count('(healthy)') >= count: + return + time.sleep(1) + raise Failure('replicas did not become healthy') + + +class Load: + def __init__(self, base, ca, concurrency): + c = Client(base, ca) + self.alice, _, _, self.chat = c.bootstrap() + self.base, self.ca, self.n = base, ca, concurrency + self.events = [] + self.lock = threading.Lock() + self.stop = threading.Event() + self.t0 = time.monotonic() + + def worker(self): + c = Client(self.base, self.ca, timeout=3) + while not self.stop.is_set(): + key = uuid.uuid4().hex + first_error = None + while not self.stop.is_set(): + try: + r = c.message(self.alice, self.chat, 'k-' + key, key=key) + t = time.monotonic() - self.t0 + with self.lock: + self.events.append((t, True, r['headers']['X-Instance-Id'], first_error)) + break + except Failure: + t = time.monotonic() - self.t0 + first_error = first_error or t + with self.lock: + self.events.append((t, False, None, None)) + time.sleep(0.05) + + def start(self): + self.pool = ThreadPoolExecutor(max_workers=self.n) + self.futures = [self.pool.submit(self.worker) for _ in range(self.n)] + + def finish(self): + self.stop.set() + self.pool.shutdown(wait=True) + + def summary(self, begin, end): + ev = [e for e in self.events if begin <= e[0] < end] + ok = [e for e in ev if e[1]] + bad = [e for e in ev if not e[1]] + span = max(end - begin, 1e-9) + return { + 'window_seconds': round(span, 2), + 'successful_requests': len(ok), + 'failed_attempts': len(bad), + 'successful_rps': round(len(ok) / span, 1), + 'last_error_offset_seconds': round(max(e[0] for e in bad) - begin, 2) if bad else None, + 'instances': sorted({e[2] for e in ok}), + } + + def instances_between(self, begin, end=float('inf')): + return sorted({e[2] for e in self.events if e[1] and begin <= e[0] < end}) + + +def failover_each(args): + ids = api_containers() + check(len(ids) >= 2, 'need at least two replicas') + load = Load(args.base_url, args.ca_file, args.concurrency) + load.start() + time.sleep(args.warmup) + results = [] + base = load.summary(2, args.warmup) + for target in ids: + before = time.monotonic() - load.t0 + sh('docker', 'stop', '-t', '10', target) + stopped = time.monotonic() - load.t0 + time.sleep(args.hold) + after_stop = time.monotonic() - load.t0 + window = load.summary(before, after_stop) + sh('docker', 'start', target) + wait_healthy(len(ids)) + restored = time.monotonic() - load.t0 + time.sleep(args.warmup) + results.append({ + 'stopped_replica': target, + 'during_failure': window, + 'seen_stopped_replica_after_stop': target in load.instances_between(stopped + 0.5, after_stop), + 'stop_command_seconds': round(stopped - before, 2), + 'restore_seconds': round(restored - after_stop, 2), + 'after_restore': load.summary(restored + 1, restored + args.warmup), + }) + load.finish() + return {'baseline': base, 'replicas': results} + + +def rolling(args): + old = api_containers() + check(len(old) >= 2, 'need at least two replicas') + n = len(old) + load = Load(args.base_url, args.ca_file, args.concurrency) + load.start() + time.sleep(args.warmup) + begin = time.monotonic() - load.t0 + sh('scripts/rolling-update.sh') + end = time.monotonic() - load.t0 + time.sleep(args.warmup) + load.finish() + return { + 'old_replicas': old, + 'new_replicas': api_containers(), + 'update_window': load.summary(begin, end), + 'old_replicas_served_after_update': [i for i in load.instances_between(end) if i in old], + } + + +def main(): + def configure(p): + p.add_argument('--mode', choices=['failover-each', 'rolling'], required=True) + p.add_argument('--concurrency', type=int, default=8) + p.add_argument('--warmup', type=float, default=8) + p.add_argument('--hold', type=float, default=12) + p.add_argument('--output') + args = arguments('Failure and rolling update under load', configure) + report = failover_each(args) if args.mode == 'failover-each' else rolling(args) + text = json.dumps(report, indent=2) + if args.output: + Path(args.output).write_text(text + '\n') + print(text) + + +if __name__ == '__main__': + run(main) diff --git a/tests/session_revocation.py b/tests/session_revocation.py new file mode 100644 index 0000000..2e2a80d --- /dev/null +++ b/tests/session_revocation.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import time +from client import Client, arguments, check, run +from stack import compose + + +def wait_valkey(timeout=60): + deadline = time.monotonic() + timeout + while '(healthy)' not in compose('ps', 'valkey', '--format', '{{.Status}}').stdout: + check(time.monotonic() < deadline, 'valkey did not recover') + time.sleep(1) + + +def main(): + args = arguments('Session revocation must not depend on the cache') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + c.call('GET', '/api/v1/chats', 200, actor=alice) + compose('stop', 'valkey') + try: + c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice) + finally: + compose('start', 'valkey') + wait_valkey() + c.call('GET', '/api/v1/chats', 401, actor=alice) + c.call('GET', '/api/v1/chats', 200, actor=bob) + print('PASS: token revoked while valkey was stopped stays revoked after valkey restart (401)') + + +if __name__ == '__main__': + run(main) diff --git a/tests/stack.py b/tests/stack.py new file mode 100644 index 0000000..f0b645f --- /dev/null +++ b/tests/stack.py @@ -0,0 +1,92 @@ +import os +import subprocess +import time +import uuid +from pathlib import Path +from client import Failure, check + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() + + +def compose(*args, env=None, check_rc=True): + full = dict(os.environ, **(env or {})) + return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=check_rc, + capture_output=True, text=True, env=full) + + +def psql(sql): + return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', sql).stdout.strip() + + +def broker(*args): + cmd = 'valkey-cli --no-auth-warning -a "$(sed -n "s/^requirepass //p" /run/secrets/broker_conf)" ' + ' '.join("'%s'" % a for a in args) + return compose('exec', '-T', 'broker', 'sh', '-c', cmd).stdout.strip() + + +def logs(service, since): + return compose('logs', '--no-color', '--since', since, service).stdout + + +def now_iso(): + return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) + + +def upload(c, actor, chat, data=SAMPLE, content_type='image/png', filename=None, expected=202): + filename = filename or 'f-' + uuid.uuid4().hex[:12] + '.png' + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, expected, actor=actor, + raw=data, headers={'Content-Type': content_type}, schema='Attachment' if expected == 202 else None) + + +def wait_status(c, actor, att_id, statuses, timeout=120): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + att = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json'] + if att['status'] in statuses: + return att + except Failure: + pass + time.sleep(0.5) + raise Failure('attachment %s did not reach %s' % (att_id, statuses)) + + +def wait_api(c, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + c.call('GET', '/health/ready') + return + except Failure: + time.sleep(1) + raise Failure('api did not become ready') + + +def wait_log(service, since, needle, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if needle in logs(service, since): + return + time.sleep(1) + raise Failure('%s did not log %r' % (service, needle)) + + +def job_row(att_id): + out = psql("select status, attempt, coalesce(last_error, '') from jobs where attachment_id='%s'" % att_id) + check(out, 'job not found') + status, attempt, error = out.split('|', 2) + return status, int(attempt), error + + +def variant_count(att_id): + return int(psql("select count(*) from attachment_variants where attachment_id='%s'" % att_id)) + + +def restore_worker(): + compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'WORKER_FAULT_POINT': ''}) + + +def restore_api(c): + compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': ''}) + compose('restart', 'lb') + wait_api(c) diff --git a/tests/telemetry_persistence.py b/tests/telemetry_persistence.py new file mode 100644 index 0000000..3d92225 --- /dev/null +++ b/tests/telemetry_persistence.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +import re +import time +from client import Client, arguments, check, run +from obs import loki, prom, trace, wait_loki +from stack import compose, upload, wait_status + + +def main(): + args = arguments('Telemetry history survives a restart of the storages') + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + request_id = upload(c, alice, chat)['headers']['X-Request-Id'] + rows = wait_loki('{service_name="messenger-api"} | event="request" | request_id="%s"' % request_id) + trace_id = rows[-1][1]['trace_id'] + check(re.fullmatch(r'[0-9a-f]{32}', trace_id), 'no trace_id in log') + time.sleep(15) + before = len(trace(trace_id)) + samples = prom('count_over_time(http_server_requests_total[10m])') + check(before > 1 and samples, 'nothing to compare') + compose('restart', 'prometheus', 'loki', 'tempo') + compose('up', '-d', '--wait', 'prometheus', 'loki', 'tempo', 'grafana') + check(len(trace(trace_id, 90)) == before, 'trace lost after restart') + check(wait_loki('{service_name="messenger-api"} | event="request" | request_id="%s"' % request_id, 90), 'log lost after restart') + check(prom('count_over_time(http_server_requests_total[10m])'), 'metrics lost after restart') + print('persistence: trace %s (%d spans), its request log and %d metric series survived a restart of prometheus, loki and tempo' + % (trace_id, before, len(samples))) + print('PASS: telemetry history is kept on volumes') + + +if __name__ == '__main__': + run(main) diff --git a/tests/upload_memory.py b/tests/upload_memory.py new file mode 100644 index 0000000..aa88929 --- /dev/null +++ b/tests/upload_memory.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import re +import subprocess +import threading +import time +from pathlib import Path +from client import Client, Failure, arguments, check, run +from image_gen import png + +ROOT = Path(__file__).resolve().parent.parent +UNITS = {'B': 1, 'KiB': 1024, 'MiB': 1024 ** 2, 'GiB': 1024 ** 3} + + +def to_mib(text): + value, unit = re.match(r'([\d.]+)\s*([A-Za-z]+)', text).groups() + return float(value) * UNITS[unit] / 1024 ** 2 + + +def sample(stop, peaks): + proc = subprocess.Popen(['docker', 'stats', '--format', '{{.Name}} {{.MemUsage}}'], cwd=ROOT, + stdout=subprocess.PIPE, text=True) + try: + for line in proc.stdout: + line = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line).strip() + m = re.match(r'(\S+-api-\d+)\s+(\S+ ?\S+) /', line) + if m: + peaks[m.group(1)] = max(peaks.get(m.group(1), 0), to_mib(m.group(2))) + if stop.is_set(): + break + finally: + proc.terminate() + + +def main(): + def configure(p): + p.add_argument('--concurrency', type=int, default=12) + p.add_argument('--rounds', type=int, default=3) + p.add_argument('--output') + args = arguments('Concurrent upload memory experiment', configure) + check(args.concurrency >= 10, 'use at least 10 concurrent uploads') + c = Client(args.base_url, args.ca_file, timeout=60) + alice, bob, eve, chat = c.bootstrap() + data = png(1800, 1800, noise=True) + digest = hashlib.sha256(data).hexdigest() + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=load.png' + + idle, stop = {}, threading.Event() + t = threading.Thread(target=sample, args=(stop, idle)) + t.start() + time.sleep(4) + stop.set() + t.join() + + peaks, stop = {}, threading.Event() + t = threading.Thread(target=sample, args=(stop, peaks)) + t.start() + time.sleep(2) + barrier = threading.Barrier(args.concurrency) + results = [] + + def worker(_): + local = Client(args.base_url, args.ca_file, timeout=60) + barrier.wait() + out = [] + for _ in range(args.rounds): + try: + r = local.call('POST', path, 202, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment') + out.append(r['json']['sha256'] == digest) + except Failure: + out.append(False) + return out + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + for out in pool.map(worker, range(args.concurrency)): + results.extend(out) + elapsed = time.monotonic() - started + time.sleep(2) + stop.set() + t.join() + limit = 192 + report = { + 'file_bytes': len(data), + 'concurrent_uploads': args.concurrency, + 'rounds': args.rounds, + 'uploads_ok': sum(results), + 'uploads_failed': len(results) - sum(results), + 'elapsed_seconds': round(elapsed, 2), + 'api_memory_limit_mib': limit, + 'idle_memory_mib': {k: round(v, 1) for k, v in sorted(idle.items())}, + 'peak_memory_mib': {k: round(v, 1) for k, v in sorted(peaks.items())}, + 'buffering_all_files_would_need_mib': round(args.concurrency * len(data) / 1024 ** 2, 1), + } + text = json.dumps(report, indent=2) + if args.output: + Path(args.output).write_text(text + '\n') + print(text) + check(report['uploads_failed'] == 0, 'some uploads failed') + check(max(peaks.values()) < limit, 'peak memory reached the limit') + + +if __name__ == '__main__': + run(main) diff --git a/tests/variants.py b/tests/variants.py new file mode 100755 index 0000000..4c00905 --- /dev/null +++ b/tests/variants.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +import hashlib +import io +from PIL import Image +from client import Client, arguments, check, run +from stack import SAMPLE, upload, wait_status + +EXPECTED = {'thumbnail': ('image/jpeg', 'JPEG'), 'crop': ('image/jpeg', 'JPEG'), 'optimized': ('image/webp', 'WEBP')} + + +def encode(img, fmt, **kw): + buf = io.BytesIO() + img.save(buf, fmt, **kw) + return buf.getvalue() + + +def halves(w, h): + img = Image.new('RGB', (w, h), (0, 0, 255)) + img.paste((255, 0, 0), (0, 0, w // 2, h)) + return img + + +def process(c, alice, bob, chat, data, content_type='image/png'): + att = upload(c, alice, chat, data, content_type)['json'] + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + check(ready['status'] == 'ready' and ready['error_code'] is None, 'attachment failed: %s' % ready['error_code']) + check(sorted(v['name'] for v in ready['variants']) == sorted(EXPECTED), 'unexpected variant set') + original = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] + check(original == data and ready['sha256'] == hashlib.sha256(data).hexdigest(), 'original changed') + out = {} + for v in ready['variants']: + r = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + v['name'], actor=bob) + body = r['bytes'] + check(r['headers'].get_content_type() == v['content_type'] == EXPECTED[v['name']][0], v['name'] + ' content type') + check(len(body) == v['size_bytes'] and hashlib.sha256(body).hexdigest() == v['sha256'], v['name'] + ' checksum') + img = Image.open(io.BytesIO(body)) + check(img.format == EXPECTED[v['name']][1] and img.size == (v['width'], v['height']), v['name'] + ' real size/format') + check('exif' not in img.info, v['name'] + ' keeps EXIF') + img.load() + out[v['name']] = img.convert('RGB') + return att, out + + +def main(): + args = arguments('Thumbnail, crop and optimized variant rules') + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + + att, v = process(c, alice, bob, chat, SAMPLE) + check(v['thumbnail'].size == (256, 160) and v['crop'].size == (128, 128) and v['optimized'].size == (320, 200), 'fixture sizes') + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=bogus', 400, actor=bob) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 404, actor=eve) + + _, v = process(c, alice, bob, chat, encode(halves(2000, 1000), 'PNG')) + check(v['thumbnail'].size == (256, 128) and v['optimized'].size == (1280, 640) and v['crop'].size == (128, 128), 'large image sizes') + + _, v = process(c, alice, bob, chat, encode(halves(100, 50), 'PNG')) + check(v['thumbnail'].size == (100, 50) and v['optimized'].size == (100, 50), 'small image must not be upscaled') + check(v['crop'].size == (128, 128), 'crop must be 128x128') + + _, v = process(c, alice, bob, chat, encode(Image.new('RGBA', (300, 300), (0, 0, 0, 0)), 'PNG')) + for name, img in v.items(): + check(min(img.getpixel((img.width // 2, img.height // 2))) >= 250, name + ': alpha not composited on white') + + exif = Image.Exif() + exif[0x0112] = 6 + _, v = process(c, alice, bob, chat, encode(halves(300, 200), 'JPEG', quality=95, exif=exif.tobytes()), 'image/jpeg') + t = v['thumbnail'] + check(t.size == (170, 256) and v['optimized'].size == (200, 300), 'EXIF orientation not applied to dimensions') + top, bottom = t.getpixel((85, 20)), t.getpixel((85, 236)) + check(top[0] > 200 and top[2] < 60 and bottom[2] > 200 and bottom[0] < 60, 'EXIF rotation direction wrong') + print('PASS: thumbnail/crop/optimized sizes, formats, checksums, no upscale, alpha on white, EXIF orientation, no EXIF in variants') + + +if __name__ == '__main__': + run(main) diff --git a/tests/worker_load.py b/tests/worker_load.py new file mode 100755 index 0000000..97bdfd7 --- /dev/null +++ b/tests/worker_load.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +import json +import subprocess +import threading +import time +from pathlib import Path +from client import Client, arguments, check, run +from image_gen import png +from stack import SAMPLE, compose, job_row, psql, restore_worker, upload, wait_status + +LIMIT_MIB = 768 + + +def sample(stop, state): + while not stop.is_set(): + out = compose('exec', '-T', 'worker', 'grep', 'VmRSS', '/proc/1/status', check_rc=False).stdout.split() + if len(out) > 1: + state['rss'] = max(state['rss'], int(out[1]) / 1024) + n = psql("select count(*) from jobs where status = 'processing'") + state['processing'] = max(state['processing'], int(n or 0)) + time.sleep(0.2) + + +def main(): + def configure(p): + p.add_argument('--huge', type=int, default=6) + p.add_argument('--small', type=int, default=20) + p.add_argument('--output') + args = arguments('Worker concurrency, memory and time limits', configure) + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + concurrency = int(compose('exec', '-T', 'worker', 'printenv', 'WORKER_CONCURRENCY').stdout.strip()) + huge = png(6320, 6320) + + state, stop = {'rss': 0, 'processing': 0}, threading.Event() + t = threading.Thread(target=sample, args=(stop, state)) + t.start() + started = time.monotonic() + ids = [upload(c, alice, chat, huge)['json']['id'] for _ in range(args.huge)] + ids += [upload(c, alice, chat, SAMPLE)['json']['id'] for _ in range(args.small)] + accepted = time.monotonic() - started + results = [wait_status(c, bob, i, ('ready', 'failed'), 300)['status'] for i in ids] + elapsed = time.monotonic() - started + stop.set() + t.join() + container = compose('ps', '-q', 'worker').stdout.strip() + oom = subprocess.run(['docker', 'inspect', '-f', '{{.RestartCount}} {{.State.OOMKilled}}', container], + capture_output=True, text=True).stdout.split() + + compose('stop', 'worker') + slow = upload(c, alice, chat, huge)['json'] + compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'JOB_TIMEOUT_SECONDS': '1'}) + timed = wait_status(c, bob, slow['id'], ('ready', 'failed'), 120) + _, attempts, last_error = job_row(slow['id']) + restore_worker() + + report = { + 'huge_images': {'count': args.huge, 'pixels': 6320 * 6320, 'bytes': len(huge)}, + 'small_images': args.small, + 'worker_concurrency': concurrency, + 'max_processing_jobs_observed': state['processing'], + 'all_ready': results.count('ready') == len(results), + 'accept_seconds': round(accepted, 2), + 'all_ready_seconds': round(elapsed, 1), + 'worker_peak_rss_mib': round(state['rss'], 1), + 'worker_memory_limit_mib': LIMIT_MIB, + 'worker_restarts_oom': oom, + 'timeout_case': {'job_timeout_seconds': 1, 'status': timed['status'], 'error_code': timed['error_code'], + 'attempts': attempts, 'last_error': last_error}, + } + text = json.dumps(report, indent=2) + if args.output: + Path(args.output).write_text(text + '\n') + print(text) + check(report['all_ready'], 'some jobs failed under load') + check(state['processing'] <= concurrency, 'more jobs processing than WORKER_CONCURRENCY') + check(state['rss'] < LIMIT_MIB and oom == ['0', 'false'], 'worker reached memory limit') + check(timed['status'] == 'failed' and timed['error_code'] == 'retry_exhausted' and 'deadline' in last_error, + 'timeout must be a bounded transient failure') + print('PASS: bounded worker concurrency, memory below limit, job timeout enforced') + + +if __name__ == '__main__': + run(main) diff --git a/worker.go b/worker.go new file mode 100644 index 0000000..fb141f0 --- /dev/null +++ b/worker.go @@ -0,0 +1,489 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "math/rand/v2" + "os" + "os/signal" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +var errSource = errors.New("staging object missing or corrupted") + +type Worker struct { + db *pgxpool.Pool + store *Store + broker *Broker + log *slog.Logger + owner string + timeout time.Duration + lease time.Duration + backoffBase time.Duration + backoffMax time.Duration + maxBytes int64 + maxPixels int64 + fault string + busy atomic.Int64 +} + +type claimedJob struct { + id, attachmentID, owner string + attempt int + created time.Time + outcome string +} + +type source struct { + stagingKey, contentType, sha256 string + size int64 +} + +func finalKey(attachmentID string) string { + return objectPrefix + attachmentID + "/original" +} + +func variantKey(attachmentID string, spec variantSpec) string { + return fmt.Sprintf("%s%s/v%d/%s.%s", objectPrefix, attachmentID, pipelineVersion, spec.name, spec.ext) +} + +func (w *Worker) inject(point string) { + if w.fault == point { + w.log.Error("fault injected", "point", point) + os.Exit(3) + } +} + +func (w *Worker) backoff(attempt int) time.Duration { + d := w.backoffBase << (attempt - 1) + if d <= 0 || d > w.backoffMax { + d = w.backoffMax + } + return d/2 + rand.N(d/2+1) +} + +func (w *Worker) claim(ctx context.Context, jobID string) (*claimedJob, error) { + tx, err := w.db.Begin(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback(ctx) + job := &claimedJob{id: jobID, owner: w.owner} + err = tx.QueryRow(ctx, ` + update jobs set status = 'processing', lease_owner = $2, + lease_until = now() + make_interval(secs => $3), updated_at = now() + where id = $1::uuid and status = 'queued' and next_attempt_at <= now() + returning attachment_id::text, attempt, created_at`, jobID, w.owner, w.lease.Seconds()).Scan(&job.attachmentID, &job.attempt, &job.created) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if _, err := tx.Exec(ctx, `update attachments set status = 'processing', updated_at = now() where id = $1::uuid`, job.attachmentID); err != nil { + return nil, err + } + return job, tx.Commit(ctx) +} + +func (w *Worker) fetch(ctx context.Context, src source) ([]byte, error) { + obj, info, err := w.store.Get(ctx, src.stagingKey) + if err != nil { + if isNoSuchKey(err) { + return nil, errSource + } + return nil, err + } + defer obj.Close() + if info.Size != src.size || info.Size > w.maxBytes { + return nil, errSource + } + data, err := io.ReadAll(io.LimitReader(obj, w.maxBytes+1)) + if err != nil { + return nil, err + } + sum := sha256.Sum256(data) + if int64(len(data)) != src.size || hex.EncodeToString(sum[:]) != src.sha256 { + return nil, errSource + } + return data, nil +} + +func (w *Worker) execute(ctx context.Context, job *claimedJob) error { + ctx, cancel := context.WithTimeout(ctx, w.timeout) + defer cancel() + var src source + err := w.db.QueryRow(ctx, ` + select coalesce(staging_key, ''), content_type, sha256, size_bytes from attachments where id = $1::uuid`, + job.attachmentID).Scan(&src.stagingKey, &src.contentType, &src.sha256, &src.size) + if err != nil { + return err + } + data, err := w.fetch(ctx, src) + if err != nil { + return err + } + _, renderSpan := childSpan(ctx, "image render", trace.SpanKindInternal) + variants, err := renderVariants(ctx, data, w.maxPixels) + endSpan(renderSpan, err) + if err != nil { + return err + } + original := finalKey(job.attachmentID) + if err := w.store.Put(ctx, original, bytes.NewReader(data), int64(len(data)), src.contentType); err != nil { + return err + } + for _, v := range variants { + if err := w.store.Put(ctx, variantKey(job.attachmentID, v.spec), bytes.NewReader(v.data), int64(len(v.data)), v.spec.contentType); err != nil { + return err + } + } + w.inject("after_store") + + tx, err := w.db.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + tag, err := tx.Exec(ctx, ` + update jobs set status = 'ready', lease_owner = null, lease_until = null, last_error = null, updated_at = now() + where id = $1::uuid and status = 'processing' and lease_owner = $2 and attempt = $3`, job.id, job.owner, job.attempt) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + job.outcome = "lease_lost" + w.log.WarnContext(ctx, "lease lost, result discarded", "job_id", job.id, "attempt", job.attempt) + return nil + } + for _, v := range variants { + _, err := tx.Exec(ctx, ` + insert into attachment_variants(attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256) + values($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9) + on conflict (attachment_id, pipeline_version, name) do update + set object_key = excluded.object_key, content_type = excluded.content_type, size_bytes = excluded.size_bytes, + width = excluded.width, height = excluded.height, sha256 = excluded.sha256`, + job.attachmentID, pipelineVersion, v.spec.name, variantKey(job.attachmentID, v.spec), v.spec.contentType, + len(v.data), v.width, v.height, v.sha256) + if err != nil { + return err + } + } + if _, err := tx.Exec(ctx, ` + update attachments set status = 'ready', error_code = null, object_key = $2, updated_at = now() + where id = $1::uuid`, job.attachmentID, original); err != nil { + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + job.outcome = "ready" + jobTimeToReady.Record(ctx, time.Since(job.created).Seconds()) + w.log.InfoContext(ctx, "job ready", "job_id", job.id, "attachment_id", job.attachmentID, "attempt", job.attempt) + w.inject("before_ack") + if err := w.store.Remove(ctx, src.stagingKey); err != nil { + w.log.WarnContext(ctx, "staging cleanup deferred to janitor", "attachment_id", job.attachmentID, "err", err.Error()) + } + return nil +} + +func (w *Worker) settle(ctx context.Context, job *claimedJob, permanent bool, code, reason string) error { + tx, err := w.db.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + var maxAttempts int + if err := tx.QueryRow(ctx, `select max_attempts from jobs where id = $1::uuid`, job.id).Scan(&maxAttempts); err != nil { + return err + } + fence := ` where id = $1::uuid and status = 'processing' and lease_owner = $2 and attempt = $3` + if !permanent && job.attempt < maxAttempts { + delay := w.backoff(job.attempt) + tag, err := tx.Exec(ctx, ` + update jobs set status = 'queued', attempt = attempt + 1, next_attempt_at = now() + make_interval(secs => $4), + lease_owner = null, lease_until = null, last_error = $5, updated_at = now()`+fence, + job.id, job.owner, job.attempt, delay.Seconds(), reason) + if err != nil || tag.RowsAffected() == 0 { + return err + } + if _, err := tx.Exec(ctx, ` + insert into outbox(job_id, attempt, available_at, created_at) + values($1::uuid, $2, now() + make_interval(secs => $3), now())`, job.id, job.attempt+1, delay.Seconds()); err != nil { + return err + } + if _, err := tx.Exec(ctx, `update attachments set status = 'queued', updated_at = now() where id = $1::uuid`, job.attachmentID); err != nil { + return err + } + job.outcome = "retry" + w.log.WarnContext(ctx, "job retry scheduled", "job_id", job.id, "attempt", job.attempt, "next_attempt", job.attempt+1, + "delay_ms", delay.Milliseconds(), "reason", reason) + return tx.Commit(ctx) + } + if !permanent { + code = "retry_exhausted" + } + tag, err := tx.Exec(ctx, ` + update jobs set status = 'failed', lease_owner = null, lease_until = null, last_error = $4, updated_at = now()`+fence, + job.id, job.owner, job.attempt, reason) + if err != nil || tag.RowsAffected() == 0 { + return err + } + if _, err := tx.Exec(ctx, ` + update attachments set status = 'failed', error_code = $2, updated_at = now() where id = $1::uuid`, job.attachmentID, code); err != nil { + return err + } + job.outcome = "failed" + w.log.ErrorContext(ctx, "job failed", "job_id", job.id, "attempt", job.attempt, "error_code", code, "reason", reason) + return tx.Commit(ctx) +} + +func (w *Worker) handle(ctx context.Context, msg redis.XMessage) bool { + var m jobMessage + raw, _ := msg.Values["job"].(string) + if err := json.Unmarshal([]byte(raw), &m); err != nil || m.SchemaVersion != 1 || m.Type != jobType || !isUUID(m.JobID) { + w.log.Warn("invalid job message dropped", "stream_id", msg.ID) + return true + } + ctx = extractTrace(ctx, traceCarrier{m.Traceparent, m.Tracestate}) + ctx, span := tracer.Start(ctx, jobStream+" process", trace.WithSpanKind(trace.SpanKindConsumer), trace.WithAttributes( + attribute.String("messaging.system", "redis"), + attribute.String("messaging.destination.name", jobStream), + attribute.String("messaging.operation.type", "process"), + attribute.String("job.id", m.JobID), + attribute.String("attachment.id", m.AttachmentID), + attribute.Int("job.attempt", m.Attempt), + )) + started := time.Now() + w.busy.Add(1) + defer w.busy.Add(-1) + job, err := w.claim(ctx, m.JobID) + if err != nil { + w.log.WarnContext(ctx, "claim failed", "job_id", m.JobID, "err", err.Error()) + endSpan(span, err) + return false + } + if job == nil { + var status string + var attempt int + _ = w.db.QueryRow(ctx, `select status, attempt from jobs where id = $1::uuid`, m.JobID).Scan(&status, &attempt) + w.log.InfoContext(ctx, "job delivery skipped", "job_id", m.JobID, "message_attempt", m.Attempt, "status", status, "attempt", attempt) + span.SetAttributes(attribute.String("job.outcome", "skipped")) + jobsProcessed.Add(ctx, 1, metric.WithAttributes(attribute.String("job.type", jobType), attribute.String("outcome", "skipped"))) + span.End() + return true + } + err = w.execute(ctx, job) + defer func() { + out := metric.WithAttributes(attribute.String("job.type", jobType), attribute.String("outcome", job.outcome)) + jobsProcessed.Add(ctx, 1, out) + jobDuration.Record(ctx, time.Since(started).Seconds(), out) + span.SetAttributes(attribute.String("job.outcome", job.outcome)) + endSpan(span, err) + }() + if err == nil { + return true + } + permanent, code := false, "processing_failed" + switch { + case errors.Is(err, errInvalidImage): + permanent, code = true, "invalid_image" + case errors.Is(err, errSource): + permanent = true + } + cause := err + if err := w.settle(ctx, job, permanent, code, cause.Error()); err != nil { + w.log.WarnContext(ctx, "settle failed", "job_id", job.id, "err", err.Error()) + job.outcome = "error" + return false + } + return true +} + +func (w *Worker) read(ctx context.Context, consumer, id string, block time.Duration) ([]redis.XMessage, error) { + res, err := w.broker.rdb.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: jobGroup, Consumer: consumer, Streams: []string{jobStream, id}, Count: 1, Block: block, + }).Result() + if len(res) == 0 { + return nil, err + } + return res[0].Messages, err +} + +func (w *Worker) consume(ctx context.Context, consumer string) { + pending := true + nextClaim := time.Now().Add(w.lease) + for ctx.Err() == nil { + var msgs []redis.XMessage + var err error + switch { + case pending: + msgs, err = w.read(ctx, consumer, "0", -1) + pending = len(msgs) > 0 + case time.Now().After(nextClaim): + msgs, _, err = w.broker.rdb.XAutoClaim(ctx, &redis.XAutoClaimArgs{ + Stream: jobStream, Group: jobGroup, Consumer: consumer, MinIdle: w.lease, Start: "0-0", Count: 1, + }).Result() + nextClaim = time.Now().Add(15 * time.Second) + default: + msgs, err = w.read(ctx, consumer, ">", 5*time.Second) + } + if err != nil && !errors.Is(err, redis.Nil) { + if ctx.Err() != nil { + return + } + if strings.Contains(err.Error(), "NOGROUP") { + _ = w.broker.ensureGroup(ctx) + } + w.log.Warn("broker read failed", "consumer", consumer, "err", err.Error()) + time.Sleep(2 * time.Second) + continue + } + for _, msg := range msgs { + if !w.handle(context.WithoutCancel(ctx), msg) { + pending = false + continue + } + if err := w.broker.rdb.XAck(context.WithoutCancel(ctx), jobStream, jobGroup, msg.ID).Err(); err != nil { + w.log.Warn("ack failed", "stream_id", msg.ID, "err", err.Error()) + } + } + } +} + +func (w *Worker) reap(ctx context.Context) error { + rows, err := w.db.Query(ctx, ` + select id::text, attachment_id::text, attempt, coalesce(lease_owner, '') + from jobs where status = 'processing' and lease_until < now() limit 50`) + if err != nil { + return err + } + jobs, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (*claimedJob, error) { + j := &claimedJob{} + return j, row.Scan(&j.id, &j.attachmentID, &j.attempt, &j.owner) + }) + if err != nil { + return err + } + for _, j := range jobs { + w.log.Warn("lease expired", "job_id", j.id, "attempt", j.attempt) + if err := w.settle(ctx, j, false, "", "lease expired"); err != nil { + return err + } + jobsProcessed.Add(ctx, 1, metric.WithAttributes(attribute.String("job.type", jobType), attribute.String("outcome", j.outcome))) + } + return nil +} + +func runWorker(db *pgxpool.Pool, store *Store, broker *Broker, log *slog.Logger, w *Worker, concurrency int) { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + for { + err := broker.ensureGroup(ctx) + if err == nil { + break + } + log.Warn("broker not ready", "err", err.Error()) + select { + case <-ctx.Done(): + return + case <-time.After(2 * time.Second): + } + } + host, _ := os.Hostname() + var wg sync.WaitGroup + wg.Go(func() { + for ctx.Err() == nil { + if err := w.reap(ctx); err != nil && ctx.Err() == nil { + log.Warn("lease reaper failed", "err", err.Error()) + } + select { + case <-ctx.Done(): + case <-time.After(5 * time.Second): + } + } + }) + for i := range concurrency { + consumer := fmt.Sprintf("%s-%d", host, i) + wg.Go(func() { w.consume(ctx, consumer) }) + } + registerWorkerMetrics(w, concurrency) + log.Info("worker started", "owner", w.owner, "concurrency", concurrency, "lease_s", w.lease.Seconds()) + <-ctx.Done() + log.Info("worker stopping") + wg.Wait() +} + +func replayJob(ctx context.Context, db *pgxpool.Pool, attachmentID string, maxAttempts int) (string, error) { + tx, err := db.Begin(ctx) + if err != nil { + return "", err + } + defer tx.Rollback(ctx) + var jobID, status string + var attempt int + var lastErr *string + err = tx.QueryRow(ctx, `select id::text, status, attempt, last_error from jobs where attachment_id = $1::uuid for update`, + attachmentID).Scan(&jobID, &status, &attempt, &lastErr) + if errors.Is(err, pgx.ErrNoRows) { + return "not_found", nil + } + if err != nil { + return "", err + } + if status != "failed" { + return "skipped_" + status, nil + } + if _, err := tx.Exec(ctx, `insert into job_replays(job_id, previous_error, replayed_at) values($1::uuid, $2, now())`, jobID, lastErr); err != nil { + return "", err + } + if _, err := tx.Exec(ctx, ` + update jobs set status = 'queued', attempt = attempt + 1, max_attempts = attempt + $2, next_attempt_at = now(), + last_error = null, updated_at = now() + where id = $1::uuid`, jobID, maxAttempts); err != nil { + return "", err + } + if _, err := tx.Exec(ctx, `insert into outbox(job_id, attempt, available_at, created_at) values($1::uuid, $2, now(), now())`, jobID, attempt+1); err != nil { + return "", err + } + if _, err := tx.Exec(ctx, `update attachments set status = 'queued', error_code = null, updated_at = now() where id = $1::uuid`, attachmentID); err != nil { + return "", err + } + return "queued", tx.Commit(ctx) +} + +func listFailed(ctx context.Context, db *pgxpool.Pool) ([]map[string]any, error) { + rows, err := db.Query(ctx, ` + select j.attachment_id::text, j.id::text, j.attempt, coalesce(a.error_code, ''), coalesce(j.last_error, ''), j.updated_at + from jobs j join attachments a on a.id = j.attachment_id + where j.status = 'failed' order by j.updated_at`) + if err != nil { + return nil, err + } + return pgx.CollectRows(rows, func(row pgx.CollectableRow) (map[string]any, error) { + var att, job, code, reason string + var attempt int + var updated time.Time + err := row.Scan(&att, &job, &attempt, &code, &reason, &updated) + return map[string]any{"attachment_id": att, "job_id": job, "attempt": attempt, "error_code": code, + "last_error": reason, "failed_at": formatTime(updated)}, err + }) +}