feat(grafana): lab6 provision metrics, logs and traces data sources

This commit is contained in:
Голиков Maximillian Антон
2026-11-17 21:03:09 +03:00
parent 2f0906cbd7
commit ada88ee980
4 changed files with 831 additions and 0 deletions
+2
View File
@@ -399,6 +399,8 @@ services:
volumes:
- grafana_data:/var/lib/grafana
- ./infra/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- ./infra/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
depends_on:
- prometheus
- opensearch
+819
View File
@@ -0,0 +1,819 @@
{
"uid": "messenger",
"title": "Мессенджер - наблюдаемость",
"schemaVersion": 39,
"version": 2,
"refresh": "30s",
"timezone": "browser",
"time": {
"from": "now-30m",
"to": "now"
},
"panels": [
{
"id": 1,
"type": "timeseries",
"title": "RPS к API",
"description": "Суммарная частота запросов к бизнес-маршрутам всех реплик.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "reqps",
"min": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"fillOpacity": 12,
"lineWidth": 2,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum(rate(http_requests_total{service_name=\"messenger-api\",http_route=~\"/api/.*\"}[$__rate_interval]))",
"legendFormat": "Запросы",
"range": true,
"instant": false,
"editorMode": "code"
}
]
},
{
"id": 2,
"type": "timeseries",
"title": "Доля ошибок 5xx API",
"description": "Ответы 5xx среди запросов к бизнес-маршрутам. При отсутствии трафика доля не определена.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "percent",
"min": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"fillOpacity": 12,
"lineWidth": 2,
"showPoints": "never"
},
"max": 100
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "100 * (sum(rate(http_requests_total{service_name=\"messenger-api\",http_route=~\"/api/.*\",http_response_status_class=\"5xx\"}[$__rate_interval])) or (0 * sum(rate(http_requests_total{service_name=\"messenger-api\",http_route=~\"/api/.*\"}[$__rate_interval])))) / (sum(rate(http_requests_total{service_name=\"messenger-api\",http_route=~\"/api/.*\"}[$__rate_interval])) > 0)",
"legendFormat": "Ошибки 5xx",
"range": true,
"instant": false,
"editorMode": "code"
}
]
},
{
"id": 3,
"type": "timeseries",
"title": "Задержка API - p50, p95, p99",
"description": "Перцентили длительности запросов по объединённым бакетам всех реплик.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"fillOpacity": 12,
"lineWidth": 2,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route=~\"/api/.*\"}[$__rate_interval])))",
"legendFormat": "p50",
"range": true,
"instant": false,
"editorMode": "code"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route=~\"/api/.*\"}[$__rate_interval])))",
"legendFormat": "p95",
"range": true,
"instant": false,
"editorMode": "code"
},
{
"refId": "C",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route=~\"/api/.*\"}[$__rate_interval])))",
"legendFormat": "p99",
"range": true,
"instant": false,
"editorMode": "code"
}
]
},
{
"id": 4,
"type": "timeseries",
"title": "Глубина очереди RabbitMQ",
"description": "Ожидающие доставки и доставленные без подтверждения задания RabbitMQ.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"fillOpacity": 12,
"lineWidth": 2,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "rabbitmq_detailed_queue_messages{job=\"rabbitmq\",vhost=\"/\",queue=\"attachments.process\"}",
"legendFormat": "Всего",
"range": true,
"instant": false,
"editorMode": "code"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "rabbitmq_detailed_queue_messages_ready{job=\"rabbitmq\",vhost=\"/\",queue=\"attachments.process\"}",
"legendFormat": "Ожидают",
"range": true,
"instant": false,
"editorMode": "code"
},
{
"refId": "C",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "rabbitmq_detailed_queue_messages_unacked{job=\"rabbitmq\",vhost=\"/\",queue=\"attachments.process\"}",
"legendFormat": "Без подтверждения",
"range": true,
"instant": false,
"editorMode": "code"
}
]
},
{
"id": 5,
"type": "timeseries",
"title": "Возраст первого задания в очереди",
"description": "Время с отправки первого ожидающего сообщения. У сообщений, отправленных до добавления timestamp, возраст отсутствует.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"fillOpacity": 12,
"lineWidth": 2,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "clamp_min(time() - (rabbitmq_detailed_queue_head_message_timestamp{job=\"rabbitmq\",vhost=\"/\",queue=\"attachments.process\"} > 0), 0)",
"legendFormat": "Возраст",
"range": true,
"instant": false,
"editorMode": "code"
}
]
},
{
"id": 6,
"type": "timeseries",
"title": "Доступность потребителей RabbitMQ",
"description": "Доля времени, когда RabbitMQ мог сразу передать сообщение потребителю.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "percent",
"min": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"fillOpacity": 12,
"lineWidth": 2,
"showPoints": "never"
},
"max": 100
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "100 * rabbitmq_detailed_queue_consumer_capacity{job=\"rabbitmq\",vhost=\"/\",queue=\"attachments.process\"}",
"legendFormat": "Доступность",
"range": true,
"instant": false,
"editorMode": "code"
}
]
},
{
"id": 7,
"type": "timeseries",
"title": "Пулы PostgreSQL - занято, свободно, лимит",
"description": "Занятые и свободные соединения и лимит каждого пула.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"fillOpacity": 12,
"lineWidth": 2,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "db_pool_connections{state=\"used\"}",
"legendFormat": "Занято · {{service_name}} / {{service_instance_id}}",
"range": true,
"instant": false,
"editorMode": "code"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "db_pool_connections{state=\"idle\"}",
"legendFormat": "Свободно · {{service_name}} / {{service_instance_id}}",
"range": true,
"instant": false,
"editorMode": "code"
},
{
"refId": "C",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "db_pool_connections{state=\"max\"}",
"legendFormat": "Лимит · {{service_name}} / {{service_instance_id}}",
"range": true,
"instant": false,
"editorMode": "code"
}
]
},
{
"id": 8,
"type": "timeseries",
"title": "Ожидание соединений PostgreSQL",
"description": "Количество незавершённых попыток получить соединение по экземплярам.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"fillOpacity": 12,
"lineWidth": 2,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "db_pool_waiting",
"legendFormat": "{{service_name}} / {{service_instance_id}}",
"range": true,
"instant": false,
"editorMode": "code"
}
]
},
{
"id": 9,
"type": "timeseries",
"title": "Исходы попыток обработки",
"description": "Частота завершённых попыток по исходам.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"min": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"fillOpacity": 12,
"lineWidth": 2,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum(rate(jobs_processed_total{outcome=\"ready\"}[$__rate_interval]))",
"legendFormat": "Готово",
"range": true,
"instant": false,
"editorMode": "code"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum(rate(jobs_processed_total{outcome=\"failed\"}[$__rate_interval]))",
"legendFormat": "Окончательный отказ",
"range": true,
"instant": false,
"editorMode": "code"
},
{
"refId": "C",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum(rate(jobs_processed_total{outcome=\"retry\"}[$__rate_interval]))",
"legendFormat": "Повтор",
"range": true,
"instant": false,
"editorMode": "code"
},
{
"refId": "D",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum(rate(jobs_processed_total{outcome=\"error\"}[$__rate_interval]))",
"legendFormat": "Ошибка",
"range": true,
"instant": false,
"editorMode": "code"
},
{
"refId": "E",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum(rate(jobs_processed_total{outcome=\"skipped\"}[$__rate_interval]))",
"legendFormat": "Пропущено",
"range": true,
"instant": false,
"editorMode": "code"
}
]
},
{
"id": 10,
"type": "timeseries",
"title": "Длительность обработки - p50, p95, p99",
"description": "Перцентили длительности попыток обработки вложений.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"fillOpacity": 12,
"lineWidth": 2,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.5, sum by (le) (rate(job_processing_duration_seconds_bucket[$__rate_interval])))",
"legendFormat": "p50",
"range": true,
"instant": false,
"editorMode": "code"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.95, sum by (le) (rate(job_processing_duration_seconds_bucket[$__rate_interval])))",
"legendFormat": "p95",
"range": true,
"instant": false,
"editorMode": "code"
},
{
"refId": "C",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.99, sum by (le) (rate(job_processing_duration_seconds_bucket[$__rate_interval])))",
"legendFormat": "p99",
"range": true,
"instant": false,
"editorMode": "code"
}
]
},
{
"id": 11,
"type": "timeseries",
"title": "Время до готовности - p50, p95, p99",
"description": "Перцентили времени от приёма вложения до фиксации статуса готовности.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 40,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"fillOpacity": 12,
"lineWidth": 2,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.5, sum by (le) (rate(end_to_end_ready_seconds_bucket[$__rate_interval])))",
"legendFormat": "p50",
"range": true,
"instant": false,
"editorMode": "code"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.95, sum by (le) (rate(end_to_end_ready_seconds_bucket[$__rate_interval])))",
"legendFormat": "p95",
"range": true,
"instant": false,
"editorMode": "code"
},
{
"refId": "C",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.99, sum by (le) (rate(end_to_end_ready_seconds_bucket[$__rate_interval])))",
"legendFormat": "p99",
"range": true,
"instant": false,
"editorMode": "code"
}
]
},
{
"id": 12,
"title": "Сбор метрик Prometheus",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Текущее состояние опроса Collector и RabbitMQ. При доступном Collector данные API и worker зависят от отправки OTLP приложением.",
"gridPos": {
"x": 12,
"y": 40,
"h": 8,
"w": 12
},
"targets": [
{
"refId": "A",
"expr": "up{job=\"otel-collector\"}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "OTel Collector",
"instant": true,
"range": false,
"editorMode": "code"
},
{
"refId": "B",
"expr": "up{job=\"rabbitmq\"}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "RabbitMQ",
"instant": true,
"range": false,
"editorMode": "code"
}
],
"fieldConfig": {
"defaults": {
"mappings": [
{
"type": "value",
"options": {
"0": {
"text": "Ошибка сбора",
"color": "red",
"index": 0
},
"1": {
"text": "Доступен",
"color": "green",
"index": 1
}
}
}
]
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value_and_name",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
}
}
]
}
@@ -0,0 +1,8 @@
apiVersion: 1
providers:
- name: messenger
type: file
allowUiUpdates: false
options:
path: /var/lib/grafana/dashboards
@@ -8,6 +8,8 @@ datasources:
url: http://prometheus:9090
isDefault: true
editable: false
jsonData:
timeInterval: 15s
- name: OpenSearch
uid: opensearch