diff --git a/compose.yaml b/compose.yaml index ea1c2a5..d58a033 100644 --- a/compose.yaml +++ b/compose.yaml @@ -399,6 +399,8 @@ services: volumes: - grafana_data:/var/lib/grafana - ./infra/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - ./infra/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro depends_on: - prometheus - opensearch diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..46e757c --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,819 @@ +{ + "uid": "messenger", + "title": "Мессенджер - наблюдаемость", + "schemaVersion": 39, + "version": 2, + "refresh": "30s", + "timezone": "browser", + "time": { + "from": "now-30m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "RPS к API", + "description": "Суммарная частота запросов к бизнес-маршрутам всех реплик.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 12, + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(http_requests_total{service_name=\"messenger-api\",http_route=~\"/api/.*\"}[$__rate_interval]))", + "legendFormat": "Запросы", + "range": true, + "instant": false, + "editorMode": "code" + } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "Доля ошибок 5xx API", + "description": "Ответы 5xx среди запросов к бизнес-маршрутам. При отсутствии трафика доля не определена.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 12, + "lineWidth": 2, + "showPoints": "never" + }, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * (sum(rate(http_requests_total{service_name=\"messenger-api\",http_route=~\"/api/.*\",http_response_status_class=\"5xx\"}[$__rate_interval])) or (0 * sum(rate(http_requests_total{service_name=\"messenger-api\",http_route=~\"/api/.*\"}[$__rate_interval])))) / (sum(rate(http_requests_total{service_name=\"messenger-api\",http_route=~\"/api/.*\"}[$__rate_interval])) > 0)", + "legendFormat": "Ошибки 5xx", + "range": true, + "instant": false, + "editorMode": "code" + } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Задержка API - p50, p95, p99", + "description": "Перцентили длительности запросов по объединённым бакетам всех реплик.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 8, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 12, + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(http_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route=~\"/api/.*\"}[$__rate_interval])))", + "legendFormat": "p50", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route=~\"/api/.*\"}[$__rate_interval])))", + "legendFormat": "p95", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route=~\"/api/.*\"}[$__rate_interval])))", + "legendFormat": "p99", + "range": true, + "instant": false, + "editorMode": "code" + } + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "Глубина очереди RabbitMQ", + "description": "Ожидающие доставки и доставленные без подтверждения задания RabbitMQ.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 8, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 12, + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rabbitmq_detailed_queue_messages{job=\"rabbitmq\",vhost=\"/\",queue=\"attachments.process\"}", + "legendFormat": "Всего", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rabbitmq_detailed_queue_messages_ready{job=\"rabbitmq\",vhost=\"/\",queue=\"attachments.process\"}", + "legendFormat": "Ожидают", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rabbitmq_detailed_queue_messages_unacked{job=\"rabbitmq\",vhost=\"/\",queue=\"attachments.process\"}", + "legendFormat": "Без подтверждения", + "range": true, + "instant": false, + "editorMode": "code" + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Возраст первого задания в очереди", + "description": "Время с отправки первого ожидающего сообщения. У сообщений, отправленных до добавления timestamp, возраст отсутствует.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 16, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 12, + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "clamp_min(time() - (rabbitmq_detailed_queue_head_message_timestamp{job=\"rabbitmq\",vhost=\"/\",queue=\"attachments.process\"} > 0), 0)", + "legendFormat": "Возраст", + "range": true, + "instant": false, + "editorMode": "code" + } + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Доступность потребителей RabbitMQ", + "description": "Доля времени, когда RabbitMQ мог сразу передать сообщение потребителю.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 16, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 12, + "lineWidth": 2, + "showPoints": "never" + }, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * rabbitmq_detailed_queue_consumer_capacity{job=\"rabbitmq\",vhost=\"/\",queue=\"attachments.process\"}", + "legendFormat": "Доступность", + "range": true, + "instant": false, + "editorMode": "code" + } + ] + }, + { + "id": 7, + "type": "timeseries", + "title": "Пулы PostgreSQL - занято, свободно, лимит", + "description": "Занятые и свободные соединения и лимит каждого пула.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 12, + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "db_pool_connections{state=\"used\"}", + "legendFormat": "Занято · {{service_name}} / {{service_instance_id}}", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "db_pool_connections{state=\"idle\"}", + "legendFormat": "Свободно · {{service_name}} / {{service_instance_id}}", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "db_pool_connections{state=\"max\"}", + "legendFormat": "Лимит · {{service_name}} / {{service_instance_id}}", + "range": true, + "instant": false, + "editorMode": "code" + } + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "Ожидание соединений PostgreSQL", + "description": "Количество незавершённых попыток получить соединение по экземплярам.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 12, + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "db_pool_waiting", + "legendFormat": "{{service_name}} / {{service_instance_id}}", + "range": true, + "instant": false, + "editorMode": "code" + } + ] + }, + { + "id": 9, + "type": "timeseries", + "title": "Исходы попыток обработки", + "description": "Частота завершённых попыток по исходам.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "min": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 12, + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(jobs_processed_total{outcome=\"ready\"}[$__rate_interval]))", + "legendFormat": "Готово", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(jobs_processed_total{outcome=\"failed\"}[$__rate_interval]))", + "legendFormat": "Окончательный отказ", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(jobs_processed_total{outcome=\"retry\"}[$__rate_interval]))", + "legendFormat": "Повтор", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(jobs_processed_total{outcome=\"error\"}[$__rate_interval]))", + "legendFormat": "Ошибка", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(jobs_processed_total{outcome=\"skipped\"}[$__rate_interval]))", + "legendFormat": "Пропущено", + "range": true, + "instant": false, + "editorMode": "code" + } + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Длительность обработки - p50, p95, p99", + "description": "Перцентили длительности попыток обработки вложений.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 12, + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(job_processing_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p50", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(job_processing_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(job_processing_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p99", + "range": true, + "instant": false, + "editorMode": "code" + } + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Время до готовности - p50, p95, p99", + "description": "Перцентили времени от приёма вложения до фиксации статуса готовности.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 12, + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(end_to_end_ready_seconds_bucket[$__rate_interval])))", + "legendFormat": "p50", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(end_to_end_ready_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(end_to_end_ready_seconds_bucket[$__rate_interval])))", + "legendFormat": "p99", + "range": true, + "instant": false, + "editorMode": "code" + } + ] + }, + { + "id": 12, + "title": "Сбор метрик Prometheus", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Текущее состояние опроса Collector и RabbitMQ. При доступном Collector данные API и worker зависят от отправки OTLP приложением.", + "gridPos": { + "x": 12, + "y": 40, + "h": 8, + "w": 12 + }, + "targets": [ + { + "refId": "A", + "expr": "up{job=\"otel-collector\"}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "OTel Collector", + "instant": true, + "range": false, + "editorMode": "code" + }, + { + "refId": "B", + "expr": "up{job=\"rabbitmq\"}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "RabbitMQ", + "instant": true, + "range": false, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Ошибка сбора", + "color": "red", + "index": 0 + }, + "1": { + "text": "Доступен", + "color": "green", + "index": 1 + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value_and_name", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + } + } + ] +} diff --git a/infra/grafana/provisioning/dashboards/messenger.yaml b/infra/grafana/provisioning/dashboards/messenger.yaml new file mode 100644 index 0000000..b783f03 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/messenger.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 + +providers: + - name: messenger + type: file + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/observability.yaml b/infra/grafana/provisioning/datasources/observability.yaml index 108b51f..b0b05cb 100644 --- a/infra/grafana/provisioning/datasources/observability.yaml +++ b/infra/grafana/provisioning/datasources/observability.yaml @@ -8,6 +8,8 @@ datasources: url: http://prometheus:9090 isDefault: true editable: false + jsonData: + timeInterval: 15s - name: OpenSearch uid: opensearch