From 3171835fff27bf766c0cff6bd204d03bd2e31e76 Mon Sep 17 00:00:00 2001 From: aegor Date: Fri, 20 Nov 2026 13:24:31 +0300 Subject: [PATCH] feat: add queue, pool, saturation metrics, slo and alert rules --- docs/runbooks.md | 63 +++++ infra/grafana/dashboards/messenger.json | 291 ++++++++++++++++++++---- infra/prometheus/rules/messenger.yml | 68 ++++++ prisma/schema.prisma | 3 +- src/db/prisma.service.ts | 5 +- src/jobs/processor.ts | 14 +- src/relay.ts | 44 +++- src/telemetry/db-pool.ts | 52 +++++ src/telemetry/sdk.ts | 20 +- src/worker.ts | 6 + 10 files changed, 511 insertions(+), 55 deletions(-) create mode 100644 docs/runbooks.md create mode 100644 infra/prometheus/rules/messenger.yml create mode 100644 src/telemetry/db-pool.ts diff --git a/docs/runbooks.md b/docs/runbooks.md new file mode 100644 index 0000000..36de540 --- /dev/null +++ b/docs/runbooks.md @@ -0,0 +1,63 @@ +# Runbooks + +Alert rules: [infra/prometheus/rules/messenger.yml](../infra/prometheus/rules/messenger.yml). Состояние: Grafana → dashboard «Messenger overview» → панель «Alerts»; или Prometheus API `GET /api/v1/alerts` (из контейнера `tools`: `wget -qO- http://grafana:3000/api/datasources/proxy/uid/prometheus/api/v1/alerts`). Уведомления наружу не отправляются. + +## SLO + +| SLI | Формула | SLO | Окно | +| --- | --- | --- | --- | +| Доступность API | 1 − 5xx / все бизнес-запросы (без `/health/*`) | ≥ 99% | 1 ч (учебный стенд; в эксплуатации — 30 дней) | +| Задержка API | доля бизнес-запросов с длительностью ≤ 0,5 с (bucket `le="0.5"`) | ≥ 95% | 1 ч | +| Готовность вложений | доля вложений, ставших `ready` за ≤ 10 с от принятия upload (bucket `le="10"`) | ≥ 95% | 1 ч | + +## MessengerHighErrorRate + +**Условие:** доля 5xx среди бизнес-запросов за 5 минут > 5% при трафике > 0,05 rps, держится 1 минуту. При SLO 99% это сжигание бюджета ошибок в 5 раз быстрее допустимого. + +**Проверить:** +1. Панель «Ответы по классам статуса» и «p95 по маршрутам»: какой маршрут даёт 5xx. +2. Loki: `{service_name="messenger-api"} | event="http.error"` — поле `message` (`database unavailable`, `object storage unavailable`, `Internal server error`), `route`, `request_id`. По `trace_id` записи — trace в Tempo. +3. В trace: на каком span ошибка — `prisma:engine:db_query`, `S3.PutObject` или код обработчика. +4. `docker compose ps` — живы ли postgres, s3, redis; `/health/ready` реплик. + +**Действия:** восстановить зависимость (`docker compose start postgres` и т. п.); при ошибке кода — откат образа. Readiness исключает реплику без БД из HAProxy автоматически. + +**Resolved:** доля 5xx за 5 минут снова ниже 5%. + +## MessengerJobQueueLag + +**Условие:** старейшее задание в состоянии `queued` с наступившим `next_attempt_at` ждёт дольше 60 с, держится 1 минуту. Источник — таблица `attachment_jobs` (метрика `jobs_queue_oldest_age_seconds` от relay), а не брокер: она показывает и задания, сообщения которых ещё не опубликованы. + +**Проверить:** +1. Панель «Worker: занято / ёмкость». `busy = capacity` — worker насыщен (медленная обработка); метрик worker нет совсем — процесс остановлен; `busy = 0` при растущей очереди — worker не получает сообщения (брокер, relay). +2. «Задания по исходу»: много `retry` — зависимость worker (S3/БД) отвечает ошибками. +3. `docker compose ps worker relay rabbitmq`; логи worker в Loki: `{service_name="messenger-worker"}`. +4. Возраст отдельных заданий: `SELECT id, status, attempt, next_attempt_at FROM attachment_jobs WHERE status='queued' ORDER BY next_attempt_at LIMIT 10;` + +**Действия:** запустить worker (`docker compose start worker`) или добавить реплик/увеличить `WORKER_CONCURRENCY` при насыщении. Задания не теряются: при старте worker обрабатывает backlog. + +**Resolved:** backlog разобран, возраст старейшего задания < 60 с. + +## MessengerSlowApi + +**Условие:** p95 бизнес-запросов за 5 минут > 1 с при трафике > 0,05 rps, держится 2 минуты. + +**Как отличить медленную БД от медленного обработчика:** +- медленная/недоступная БД: растёт `db_client_connection_pending_requests` (ожидание пула) и/или в trace медленного запроса основное время — spans `prisma:engine:db_query`; ошибки `database unavailable` (503); +- медленный обработчик: пул свободен, spans БД короткие, время приходится на сам SERVER span или S3/рендеринг; растёт CPU API. + +**Действия:** для БД — проверить postgres (`pg_stat_activity`, блокировки, ресурсы контейнера); для обработчика — профилировать маршрут, найденный в «p95 по маршрутам». + +**Resolved:** p95 за 5 минут < 1 с. + +## TelemetryExportFailing + +**Условие:** Collector получает ошибки при отправке в Loki/Tempo/Prometheus или очередь exporter заполнена больше чем наполовину, держится 1 минуту. + +**Что происходит:** API и worker продолжают работать: SDK отправляет телеметрию асинхронно с таймаутом 5 с и ограниченными очередями (2048 записей), Collector держит ограниченную очередь (`queue_size: 200` batch на exporter), повторяет отправку до 60 с и затем отбрасывает данные. `memory_limiter` (160 MiB) отказывает в приёме при нехватке памяти. Потери видны в `otelcol_exporter_send_failed_*` и `otelcol_receiver_refused_*`. + +**Проверить:** панель «Конвейер телеметрии»; `docker compose ps loki tempo prometheus`; логи Collector `docker compose logs otel-collector`. + +**Действия:** запустить хранилище (`docker compose start loki tempo`). Данные, отброшенные за время недоступности, не восстанавливаются; логи остаются в stdout контейнеров (`docker compose logs`). + +**Resolved:** отказов отправки нет, очередь exporter опустела. diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json index d935127..5cb751d 100644 --- a/infra/grafana/dashboards/messenger.json +++ b/infra/grafana/dashboards/messenger.json @@ -4,7 +4,7 @@ "tags": ["messenger", "lab6"], "timezone": "browser", "schemaVersion": 39, - "version": 1, + "version": 2, "editable": false, "graphTooltip": 1, "refresh": "10s", @@ -12,19 +12,87 @@ "templating": { "list": [] }, "annotations": { "list": [] }, "panels": [ + { "type": "row", "title": "SLO и alerts", "id": 100, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } }, { - "type": "row", - "title": "API", - "id": 100, - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } + "type": "stat", + "title": "Доступность API, 1 ч (SLO ≥ 99% без 5xx)", + "id": 20, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", "decimals": 2, + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0.99 }] } + }, + "overrides": [] + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background" }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "1 - messenger:http_error_ratio:rate1h" }] }, + { + "type": "stat", + "title": "Запросы быстрее 500 мс, 1 ч (SLO ≥ 95%)", + "id": 21, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", "decimals": 2, + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0.95 }] } + }, + "overrides": [] + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background" }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "messenger:http_fast_ratio:rate1h" }] + }, + { + "type": "stat", + "title": "Вложения готовы за ≤ 10 с, 1 ч (SLO ≥ 95%)", + "id": 22, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", "decimals": 2, + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0.95 }] } + }, + "overrides": [] + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background" }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "messenger:time_to_ready_fast_ratio:rate1h" }] + }, + { + "type": "state-timeline", + "title": "Alerts (pending / firing)", + "id": 23, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 2 }] }, + "mappings": [{ "type": "value", "options": { "1": { "text": "pending" }, "2": { "text": "firing" } } }] + }, + "overrides": [] + }, + "options": { "showValue": "never", "mergeValues": true, "legend": { "showLegend": false } }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "max by (alertname) (ALERTS{alertstate=\"firing\"}) * 2 or max by (alertname) (ALERTS{alertstate=\"pending\"})", + "legendFormat": "{{alertname}}" + } + ] + }, + + { "type": "row", "title": "API", "id": 101, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 } }, { "type": "timeseries", "title": "RPS по маршрутам (без health)", "id": 1, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull"] } }, "targets": [ @@ -41,7 +109,7 @@ "title": "Ответы по классам статуса (без health)", "id": 2, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, "targets": [ { @@ -54,10 +122,10 @@ }, { "type": "timeseries", - "title": "Доля 5xx среди бизнес-запросов (окно 5m)", + "title": "Доля 5xx среди бизнес-запросов (окно 5m, порог alert 5%)", "id": 3, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 9 }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 15 }, "fieldConfig": { "defaults": { "unit": "percentunit", @@ -68,12 +136,7 @@ "overrides": [] }, "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "(sum(rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\", http_response_status_class=\"5xx\"}[5m])) or vector(0)) / sum(rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\"}[5m]))", - "legendFormat": "5xx ratio" - } + { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "messenger:http_error_ratio:rate5m", "legendFormat": "5xx ratio" } ] }, { @@ -81,7 +144,7 @@ "title": "Длительность API p50 / p95 / p99 (все реплики, без health)", "id": 4, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 9 }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 15 }, "fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] }, "targets": [ { @@ -109,7 +172,7 @@ "title": "p95 по маршрутам", "id": 5, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 9 }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 15 }, "fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] }, "targets": [ { @@ -120,19 +183,61 @@ } ] }, + + { "type": "row", "title": "Очередь и worker", "id": 102, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 } }, { - "type": "row", - "title": "Worker", - "id": 101, - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 } + "type": "timeseries", + "title": "Глубина очереди attachment.jobs (из attachment_jobs)", + "id": 30, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 24 }, + "fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "max by (state) (jobs_queue_depth{job=\"messenger-relay\"})", + "legendFormat": "{{state}}" + } + ] + }, + { + "type": "timeseries", + "title": "Возраст старейшего готового задания (порог alert 60 s)", + "id": 31, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 24 }, + "fieldConfig": { + "defaults": { + "unit": "s", "min": 0, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 60 }] }, + "custom": { "thresholdsStyle": { "mode": "line" } } + }, + "overrides": [] + }, + "targets": [ + { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "messenger:jobs_queue_oldest_age:max", "legendFormat": "oldest due job age" } + ] + }, + { + "type": "timeseries", + "title": "Worker: занято / ёмкость", + "id": 32, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 24 }, + "fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] }, + "targets": [ + { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(worker_jobs_busy{job=\"messenger-worker\"})", "legendFormat": "busy" }, + { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(worker_jobs_capacity{job=\"messenger-worker\"})", "legendFormat": "capacity" }, + { "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(avg_over_time(worker_jobs_busy{job=\"messenger-worker\"}[5m])) / sum(worker_jobs_capacity{job=\"messenger-worker\"})", "legendFormat": "saturation 5m" } + ] }, { "type": "timeseries", "title": "Задания по исходу (в секунду)", "id": 6, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 32 }, "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, "targets": [ { @@ -148,7 +253,7 @@ "title": "Длительность обработки задания p50 / p95 / p99", "id": 7, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 32 }, "fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] }, "targets": [ { @@ -168,35 +273,127 @@ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome=\"success\"}[$__rate_interval])))", "legendFormat": "p99 success" - }, - { - "refId": "D", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome!=\"success\"}[$__rate_interval])))", - "legendFormat": "p95 retry/failed" } ] }, { - "type": "row", - "title": "Логи и трассы", - "id": 102, - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 } + "type": "timeseries", + "title": "Время от upload до ready p50 / p95 / p99", + "id": 33, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 32 }, + "fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(attachments_time_to_ready_seconds_bucket{job=\"messenger-worker\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(attachments_time_to_ready_seconds_bucket{job=\"messenger-worker\"}[$__rate_interval])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(attachments_time_to_ready_seconds_bucket{job=\"messenger-worker\"}[$__rate_interval])))", + "legendFormat": "p99" + } + ] }, + + { "type": "row", "title": "БД и ресурсы", "id": 103, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 } }, + { + "type": "timeseries", + "title": "Пул БД: занято / ожидают соединения", + "id": 40, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 41 }, + "fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] }, + "targets": [ + { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (job) (db_client_connection_count{db_client_connection_state=\"used\"})", "legendFormat": "{{job}} used" }, + { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (job) (db_client_connection_pending_requests)", "legendFormat": "{{job}} waiting" }, + { "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (job) (db_client_connection_max)", "legendFormat": "{{job}} max" } + ] + }, + { + "type": "timeseries", + "title": "CPU процессов (доля ядра)", + "id": 41, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 41 }, + "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0 }, "overrides": [] }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum by (job, instance) (rate(process_cpu_time_seconds_total{job=~\"messenger-.+\"}[$__rate_interval]))", + "legendFormat": "{{job}} {{instance}}" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(otelcol_process_cpu_seconds{job=\"otel-collector\"}[$__rate_interval])", + "legendFormat": "otel-collector" + } + ] + }, + { + "type": "timeseries", + "title": "RAM процессов (RSS)", + "id": 42, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 41 }, + "fieldConfig": { "defaults": { "unit": "bytes", "min": 0 }, "overrides": [] }, + "targets": [ + { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "process_memory_usage_bytes{job=~\"messenger-.+\"}", "legendFormat": "{{job}} {{instance}}" }, + { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "otelcol_process_memory_rss{job=\"otel-collector\"}", "legendFormat": "otel-collector" } + ] + }, + + { "type": "row", "title": "Конвейер телеметрии (Collector)", "id": 104, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 49 } }, + { + "type": "timeseries", + "title": "Принято / отправлено / ошибки отправки (в секунду)", + "id": 50, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 50 }, + "fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] }, + "targets": [ + { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_receiver_accepted_spans[$__rate_interval]))", "legendFormat": "spans accepted" }, + { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_exporter_sent_spans[$__rate_interval]))", "legendFormat": "spans sent" }, + { "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_exporter_send_failed_spans[$__rate_interval])) or vector(0)", "legendFormat": "spans failed" }, + { "refId": "D", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_exporter_sent_log_records[$__rate_interval]))", "legendFormat": "logs sent" }, + { "refId": "E", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_exporter_send_failed_log_records[$__rate_interval])) or vector(0)", "legendFormat": "logs failed" }, + { "refId": "F", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_receiver_refused_spans[$__rate_interval])) + sum(rate(otelcol_receiver_refused_log_records[$__rate_interval])) or vector(0)", "legendFormat": "refused (memory_limiter)" } + ] + }, + { + "type": "timeseries", + "title": "Очередь exporter (ограничена queue_size)", + "id": 51, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 50 }, + "fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] }, + "targets": [ + { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "max by (exporter) (otelcol_exporter_queue_size)", "legendFormat": "{{exporter}} size" }, + { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "max(otelcol_exporter_queue_capacity)", "legendFormat": "capacity" } + ] + }, + + { "type": "row", "title": "Логи и трассы", "id": 105, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 58 } }, { "type": "logs", "title": "Логи API и worker (WARN и выше)", "id": 8, "datasource": { "type": "loki", "uid": "loki" }, - "gridPos": { "h": 10, "w": 12, "x": 0, "y": 27 }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 59 }, "options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true }, "targets": [ - { - "refId": "A", - "datasource": { "type": "loki", "uid": "loki" }, - "expr": "{service_name=~\"messenger-.+\"} | severity_text=~\"WARN|ERROR|FATAL\"" - } + { "refId": "A", "datasource": { "type": "loki", "uid": "loki" }, "expr": "{service_name=~\"messenger-.+\"} | severity_text=~\"WARN|ERROR|FATAL\"" } ] }, { @@ -204,14 +401,10 @@ "title": "Все логи (http.request, job.handled, …)", "id": 9, "datasource": { "type": "loki", "uid": "loki" }, - "gridPos": { "h": 10, "w": 12, "x": 12, "y": 27 }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 59 }, "options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true }, "targets": [ - { - "refId": "A", - "datasource": { "type": "loki", "uid": "loki" }, - "expr": "{service_name=~\"messenger-.+\"}" - } + { "refId": "A", "datasource": { "type": "loki", "uid": "loki" }, "expr": "{service_name=~\"messenger-.+\"}" } ] }, { @@ -219,7 +412,7 @@ "title": "Последние трассы (HTTP server и consumer spans)", "id": 10, "datasource": { "type": "tempo", "uid": "tempo" }, - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 37 }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 69 }, "targets": [ { "refId": "A", diff --git a/infra/prometheus/rules/messenger.yml b/infra/prometheus/rules/messenger.yml new file mode 100644 index 0000000..6025dfb --- /dev/null +++ b/infra/prometheus/rules/messenger.yml @@ -0,0 +1,68 @@ +groups: + - name: messenger-sli + interval: 15s + rules: + - record: messenger:http_requests:rate5m + expr: sum(rate(http_server_requests_total{job="messenger-api", http_route!~"/health.*"}[5m])) + - record: messenger:http_errors:rate5m + expr: sum(rate(http_server_requests_total{job="messenger-api", http_route!~"/health.*", http_response_status_class="5xx"}[5m])) or vector(0) + - record: messenger:http_error_ratio:rate5m + expr: messenger:http_errors:rate5m / messenger:http_requests:rate5m + - record: messenger:http_error_ratio:rate1h + expr: | + (sum(rate(http_server_requests_total{job="messenger-api", http_route!~"/health.*", http_response_status_class="5xx"}[1h])) or vector(0)) + / sum(rate(http_server_requests_total{job="messenger-api", http_route!~"/health.*"}[1h])) + - record: messenger:http_latency_p95:rate5m + expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health.*"}[5m]))) + - record: messenger:http_fast_ratio:rate1h + expr: | + sum(rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health.*", le="0.5"}[1h])) + / sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health.*"}[1h])) + - record: messenger:time_to_ready_fast_ratio:rate1h + expr: | + sum(rate(attachments_time_to_ready_seconds_bucket{job="messenger-worker", le="10"}[1h])) + / sum(rate(attachments_time_to_ready_seconds_count{job="messenger-worker"}[1h])) + - record: messenger:jobs_queue_oldest_age:max + expr: max(jobs_queue_oldest_age_seconds{job="messenger-relay"}) + + - name: messenger-alerts + rules: + - alert: MessengerHighErrorRate + expr: messenger:http_error_ratio:rate5m > 0.05 and messenger:http_requests:rate5m > 0.05 + for: 1m + labels: + severity: critical + annotations: + summary: "Доля 5xx бизнес-запросов API {{ $value | humanizePercentage }} за 5 минут (порог 5%)" + description: "SLO доступности 99% за 1 ч: при 5% ошибок бюджет сгорает в 5 раз быстрее допустимого. Health-запросы не учитываются." + runbook: "docs/runbooks.md#messengerhigherrorrate" + - alert: MessengerJobQueueLag + expr: messenger:jobs_queue_oldest_age:max > 60 + for: 1m + labels: + severity: warning + annotations: + summary: "Старейшее готовое к обработке задание ждёт {{ $value | humanizeDuration }} (порог 60 s)" + description: "Задания принимаются, но не обрабатываются: worker остановлен, не справляется или не получает сообщения из брокера." + runbook: "docs/runbooks.md#messengerjobqueuelag" + - alert: MessengerSlowApi + expr: messenger:http_latency_p95:rate5m > 1 and messenger:http_requests:rate5m > 0.05 + for: 2m + labels: + severity: warning + annotations: + summary: "p95 бизнес-запросов API {{ $value | humanizeDuration }} за 5 минут (порог 1 s)" + description: "Проверьте ожидание пула БД (db_client_connection_pending_requests) и spans prisma:engine:db_query в трассах медленных запросов." + runbook: "docs/runbooks.md#messengerslowapi" + - alert: TelemetryExportFailing + expr: | + sum(rate(otelcol_exporter_send_failed_spans[2m])) + sum(rate(otelcol_exporter_send_failed_log_records[2m])) + + sum(rate(otelcol_exporter_send_failed_metric_points[2m])) > 0 + or max(otelcol_exporter_queue_size / otelcol_exporter_queue_capacity) > 0.5 + for: 1m + labels: + severity: warning + annotations: + summary: "Collector не может доставить телеметрию в хранилище" + description: "Растёт очередь exporter или есть отказы отправки. Мессенджер работает, но часть трасс/логов теряется после заполнения ограниченной очереди." + runbook: "docs/runbooks.md#telemetryexportfailing" diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 2942712..03d6fb8 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -1,5 +1,6 @@ generator client { - provider = "prisma-client-js" + provider = "prisma-client-js" + previewFeatures = ["metrics"] } datasource db { diff --git a/src/db/prisma.service.ts b/src/db/prisma.service.ts index e391433..bcca5b9 100644 --- a/src/db/prisma.service.ts +++ b/src/db/prisma.service.ts @@ -1,13 +1,16 @@ import { Injectable, OnModuleDestroy } from '@nestjs/common'; import { Prisma, PrismaClient } from '@prisma/client'; import { readFileSync } from 'fs'; +import { registerDbPool } from '../telemetry/db-pool'; const UNAVAILABLE_CODES = new Set(['P1001', 'P1002', 'P1008', 'P1017', 'P2024']); @Injectable() export class PrismaService extends PrismaClient implements OnModuleDestroy { constructor() { - super({ datasourceUrl: readDatabaseUrl() }); + const url = readDatabaseUrl(); + super({ datasourceUrl: url }); + registerDbPool(this, url); } async onModuleDestroy(): Promise { diff --git a/src/jobs/processor.ts b/src/jobs/processor.ts index 9491ee5..79576da 100644 --- a/src/jobs/processor.ts +++ b/src/jobs/processor.ts @@ -1,5 +1,5 @@ import { Logger } from '@nestjs/common'; -import { SpanStatusCode, trace } from '@opentelemetry/api'; +import { metrics, SpanStatusCode, trace } from '@opentelemetry/api'; import { Attachment, AttachmentJob, PrismaClient } from '@prisma/client'; import { createHash, randomUUID } from 'crypto'; import { @@ -7,7 +7,7 @@ import { ObjectTooLargeError, S3Service, } from '../storage/s3.service'; -import { JobMessage, PIPELINE_VERSION } from './job-message'; +import { JOB_TYPE, JobMessage, PIPELINE_VERSION } from './job-message'; import { markFailed, retryOrFail } from './job-store'; import { InvalidImageError, @@ -52,6 +52,13 @@ class PermanentError extends Error { class TimeoutError extends Error {} const tracer = trace.getTracer('messenger-worker'); +const timeToReady = metrics.getMeter('messenger-worker').createHistogram('attachments.time_to_ready', { + description: 'From durable upload acceptance to the committed ready state', + unit: 's', + advice: { + explicitBucketBoundaries: [0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120, 300, 600], + }, +}); export class JobProcessor { private readonly logger = new Logger('JobProcessor'); @@ -225,6 +232,9 @@ export class JobProcessor { this.logger.warn(`job ${lease.id}: lease lost before commit, result left to owner`); return 'skipped'; } + timeToReady.record((now.getTime() - attachment.createdAt.getTime()) / 1000, { + 'job.type': JOB_TYPE, + }); await this.hooks.afterCommit?.(); await this.removeStaging(attachment); return 'ready'; diff --git a/src/relay.ts b/src/relay.ts index bcc43d6..c7467f8 100644 --- a/src/relay.ts +++ b/src/relay.ts @@ -1,6 +1,14 @@ import './telemetry/register'; import { Logger } from '@nestjs/common'; -import { propagation, ROOT_CONTEXT, Span, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api'; +import { + metrics, + propagation, + ROOT_CONTEXT, + Span, + SpanKind, + SpanStatusCode, + trace, +} from '@opentelemetry/api'; import { ConfirmChannel } from 'amqplib'; import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker'; import { PrismaService } from './db/prisma.service'; @@ -22,6 +30,40 @@ const prisma = new PrismaService(); const tracer = trace.getTracer('messenger-relay'); let stopping = false; +const meter = metrics.getMeter('messenger-relay'); +const queueDepth = meter.createObservableGauge('jobs.queue.depth', { + description: 'Attachment jobs by state (source of truth: attachment_jobs)', +}); +const oldestAge = meter.createObservableGauge('jobs.queue.oldest_age', { + description: 'Age of the oldest due queued job (since accepted or retry became due)', + unit: 's', +}); + +meter.addBatchObservableCallback( + async (result) => { + const [row] = await prisma.$queryRaw< + { queued: number; due: number; processing: number; oldest: number }[] + >` + SELECT + count(*) FILTER (WHERE status = 'queued')::int AS queued, + count(*) FILTER (WHERE status = 'queued' AND next_attempt_at <= now())::int AS due, + count(*) FILTER (WHERE status = 'processing')::int AS processing, + COALESCE(EXTRACT(EPOCH FROM now() - min(next_attempt_at) + FILTER (WHERE status = 'queued' AND next_attempt_at <= now())), 0)::float8 AS oldest + FROM attachment_jobs + WHERE status IN ('queued', 'processing')`.catch(() => []); + if (!row) { + return; + } + const queue = { 'messaging.destination.name': JOB_QUEUE }; + result.observe(queueDepth, row.queued, { ...queue, state: 'queued' }); + result.observe(queueDepth, row.due, { ...queue, state: 'due' }); + result.observe(queueDepth, row.processing, { ...queue, state: 'processing' }); + result.observe(oldestAge, row.oldest, queue); + }, + [queueDepth, oldestAge], +); + function startPublishSpan(payload: JobMessage, headers: Record): Span { const parent = propagation.extract(ROOT_CONTEXT, payload); const span = tracer.startSpan( diff --git a/src/telemetry/db-pool.ts b/src/telemetry/db-pool.ts new file mode 100644 index 0000000..6d280fa --- /dev/null +++ b/src/telemetry/db-pool.ts @@ -0,0 +1,52 @@ +import { metrics } from '@opentelemetry/api'; +import { PrismaClient } from '@prisma/client'; + +interface Pool { + client: PrismaClient; + max: number; +} + +const pools: Pool[] = []; +const meter = metrics.getMeter('messenger-db'); +const connections = meter.createObservableGauge('db.client.connection.count', { + description: 'Prisma pool connections by state', +}); +const pending = meter.createObservableGauge('db.client.connection.pending_requests', { + description: 'Queries waiting for a free pool connection', +}); +const max = meter.createObservableGauge('db.client.connection.max', { + description: 'Configured pool size (connection_limit)', +}); + +meter.addBatchObservableCallback( + async (result) => { + for (const pool of pools) { + let gauges: Record; + try { + const snapshot = await pool.client.$metrics.json(); + gauges = Object.fromEntries(snapshot.gauges.map((gauge) => [gauge.key, gauge.value])); + } catch { + continue; + } + result.observe(connections, gauges.prisma_pool_connections_busy ?? 0, { + 'db.client.connection.state': 'used', + }); + result.observe(connections, gauges.prisma_pool_connections_idle ?? 0, { + 'db.client.connection.state': 'idle', + }); + result.observe(pending, gauges.prisma_client_queries_wait ?? 0); + result.observe(max, pool.max); + } + }, + [connections, pending, max], +); + +export function registerDbPool(client: PrismaClient, url: string): void { + let limit = 0; + try { + limit = Number(new URL(url).searchParams.get('connection_limit') ?? 0); + } catch { + limit = 0; + } + pools.push({ client, max: limit }); +} diff --git a/src/telemetry/sdk.ts b/src/telemetry/sdk.ts index 96cebbc..46a2e32 100644 --- a/src/telemetry/sdk.ts +++ b/src/telemetry/sdk.ts @@ -1,4 +1,4 @@ -import { Attributes, Context, Link, SpanKind } from '@opentelemetry/api'; +import { Attributes, Context, Link, metrics, SpanKind } from '@opentelemetry/api'; import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-proto'; import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto'; import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto'; @@ -133,6 +133,24 @@ export function startTelemetry(): void { ], }); sdk.start(); + registerProcessMetrics(); +} + +function registerProcessMetrics(): void { + const meter = metrics.getMeter('messenger-process'); + meter + .createObservableCounter('process.cpu.time', { description: 'Process CPU time', unit: 's' }) + .addCallback((result) => { + const usage = process.cpuUsage(); + result.observe(usage.user / 1e6, { 'cpu.mode': 'user' }); + result.observe(usage.system / 1e6, { 'cpu.mode': 'system' }); + }); + meter + .createObservableGauge('process.memory.usage', { description: 'Resident set size', unit: 'By' }) + .addCallback((result) => result.observe(process.memoryUsage.rss())); + meter + .createObservableGauge('process.memory.heap_used', { description: 'V8 heap used', unit: 'By' }) + .addCallback((result) => result.observe(process.memoryUsage().heapUsed)); } export async function shutdownTelemetry(): Promise { diff --git a/src/worker.ts b/src/worker.ts index 9dab179..600f681 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -45,6 +45,12 @@ const jobDuration = meter.createHistogram('jobs.duration', { unit: 's', advice: { explicitBucketBoundaries: JOB_BUCKETS }, }); +meter + .createObservableGauge('worker.jobs.busy', { description: 'Deliveries being handled now' }) + .addCallback((result) => result.observe(inFlight)); +meter + .createObservableGauge('worker.jobs.capacity', { description: 'Concurrent deliveries allowed (prefetch)' }) + .addCallback((result) => result.observe(CONCURRENCY)); function parentContext(delivery: ConsumeMessage, message: JobMessage | null): Context { const headers = delivery.properties.headers ?? {};