ops: add observability stack and SLO alerts
This commit is contained in:
+602
-8
@@ -67,7 +67,7 @@
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "API latency p95 by route",
|
||||
"title": "API latency p50 / p95 / p99 by route",
|
||||
"id": 3,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
@@ -77,9 +77,19 @@
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "{{method}} {{route}}",
|
||||
"expr": "histogram_quantile(0.50, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "p50 {{method}} {{route}}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "p95 {{method}} {{route}}",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "p99 {{method}} {{route}}",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
@@ -895,14 +905,563 @@
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "SLO and alerts",
|
||||
"id": 105,
|
||||
"collapsed": true,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 109
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Business API availability",
|
||||
"id": 31,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 110
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "1 - (sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001)",
|
||||
"legendFormat": "availability",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "API error-budget burn rate (99.5% SLO)",
|
||||
"id": 32,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 110
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001) / 0.005",
|
||||
"legendFormat": "5m burn",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[1h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[1h])) or vector(0)), 0.000001) / 0.005",
|
||||
"legendFormat": "1h burn",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[6h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[6h])) or vector(0)), 0.000001) / 0.005",
|
||||
"legendFormat": "6h burn",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Attachments ready within 30 seconds",
|
||||
"id": 33,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 118
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "1 - (clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001)",
|
||||
"legendFormat": "ready in 30s",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Attachment ready-time error-budget burn",
|
||||
"id": 34,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 118
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001) / 0.01",
|
||||
"legendFormat": "5m burn",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[1h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0)), 0.000001) / 0.01",
|
||||
"legendFormat": "1h burn",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[6h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0)), 0.000001) / 0.01",
|
||||
"legendFormat": "6h burn",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "SLO alert state",
|
||||
"id": 35,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 126
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max by (alertname, alertstate) (ALERTS{alertname=~\"MessengerAPIHighErrorBudgetBurn.*|AttachmentReadyHighErrorBudgetBurn.*|AttachmentQueueBacklogGrowing|AttachmentWorkerUnavailable\"})",
|
||||
"legendFormat": "{{alertname}} {{alertstate}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Runtime resources",
|
||||
"id": 107,
|
||||
"collapsed": true,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 134
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Process CPU by service",
|
||||
"id": 38,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 135
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (service_name) (rate(process_cpu_seconds_total[1m]))",
|
||||
"legendFormat": "{{service_name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "cores"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Go heap by service",
|
||||
"id": 39,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 135
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (service_name) (go_memstats_heap_alloc_bytes)",
|
||||
"legendFormat": "{{service_name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Database pool used and wait rate",
|
||||
"id": 40,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 143
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (service_name) (messenger_db_pool_connections{state=\"used\"})",
|
||||
"legendFormat": "{{service_name}} used",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (service_name) (rate(messenger_db_pool_waited_total[5m]))",
|
||||
"legendFormat": "{{service_name}} waits/s",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Worker saturation",
|
||||
"id": 41,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 143
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(messenger_attachment_worker_jobs_in_flight)",
|
||||
"legendFormat": "busy",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum(messenger_attachment_worker_capacity)",
|
||||
"legendFormat": "capacity",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Logs and traces",
|
||||
"id": 106,
|
||||
"collapsed": true,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 151
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "logs",
|
||||
"title": "Correlated application logs",
|
||||
"id": 36,
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 152
|
||||
},
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "{service_name=~\"messenger-.+\"} | json",
|
||||
"queryType": "range",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"dedupStrategy": "none",
|
||||
"enableLogDetails": true,
|
||||
"prettifyLogMessage": false,
|
||||
"showCommonLabels": false,
|
||||
"showLabels": false,
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "traces",
|
||||
"title": "Recent application traces (TraceQL)",
|
||||
"id": 37,
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 152
|
||||
},
|
||||
"datasource": {
|
||||
"type": "tempo",
|
||||
"uid": "tempo"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "tempo",
|
||||
"uid": "tempo"
|
||||
},
|
||||
"query": "{ resource.service.name =~ \"messenger-.*\" }",
|
||||
"queryType": "traceql",
|
||||
"refId": "A",
|
||||
"limit": 50,
|
||||
"tableType": "traces"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showSpanFilters": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Trace-derived service call rate",
|
||||
"id": 42,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 162
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (service, span_name) (rate(traces_spanmetrics_calls_total[5m]))",
|
||||
"legendFormat": "{{service}} {{span_name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Collector delivery failures and queue pressure",
|
||||
"id": 43,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 162
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans_total[5m]))",
|
||||
"legendFormat": "{{exporter}} spans failed/s",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records_total[5m]))",
|
||||
"legendFormat": "{{exporter}} logs failed/s",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "max by (exporter) (otelcol_exporter_queue_size)",
|
||||
"legendFormat": "{{exporter}} queue",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "max by (exporter) (otelcol_exporter_queue_capacity)",
|
||||
"legendFormat": "{{exporter}} capacity",
|
||||
"refId": "D"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"refresh": "5s",
|
||||
"schemaVersion": 41,
|
||||
"tags": [
|
||||
"highload",
|
||||
"lab5",
|
||||
"pipeline"
|
||||
"lab6",
|
||||
"observability",
|
||||
"slo"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
@@ -912,7 +1471,42 @@
|
||||
"to": "now"
|
||||
},
|
||||
"timezone": "browser",
|
||||
"title": "Messenger lab5 overview",
|
||||
"uid": "messenger-lab5",
|
||||
"version": 3
|
||||
"title": "Messenger lab6 observability",
|
||||
"uid": "messenger-lab6",
|
||||
"version": 1,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": false,
|
||||
"icon": "external link",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": [],
|
||||
"targetBlank": false,
|
||||
"title": "Explore logs and traces",
|
||||
"type": "link",
|
||||
"url": "/explore"
|
||||
},
|
||||
{
|
||||
"asDropdown": false,
|
||||
"icon": "external link",
|
||||
"includeVars": false,
|
||||
"keepTime": false,
|
||||
"tags": [],
|
||||
"targetBlank": true,
|
||||
"title": "API runbook",
|
||||
"type": "link",
|
||||
"url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md"
|
||||
},
|
||||
{
|
||||
"asDropdown": false,
|
||||
"icon": "external link",
|
||||
"includeVars": false,
|
||||
"keepTime": false,
|
||||
"tags": [],
|
||||
"targetBlank": true,
|
||||
"title": "Pipeline runbook",
|
||||
"type": "link",
|
||||
"url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
apiVersion: 1
|
||||
|
||||
deleteDatasources:
|
||||
- name: Prometheus
|
||||
orgId: 1
|
||||
- name: Loki
|
||||
orgId: 1
|
||||
- name: Tempo
|
||||
orgId: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
exemplarTraceIdDestinations:
|
||||
- name: trace_id
|
||||
datasourceUid: tempo
|
||||
|
||||
- name: Loki
|
||||
type: loki
|
||||
uid: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
maxLines: 1000
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherRegex: 'trace_id[=:" ]+([0-9a-f]{32})'
|
||||
datasourceUid: tempo
|
||||
url: '$${__value.raw}'
|
||||
|
||||
- name: Tempo
|
||||
type: tempo
|
||||
uid: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
search:
|
||||
hide: false
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: -5m
|
||||
spanEndTimeShift: 5m
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
customQuery: true
|
||||
query: '{service_name="$${__span.tags["service.name"]}"} | trace_id = "$${__trace.traceId}"'
|
||||
tags:
|
||||
- key: service.name
|
||||
value: service_name
|
||||
tracesToMetrics:
|
||||
datasourceUid: prometheus
|
||||
spanStartTimeShift: -5m
|
||||
spanEndTimeShift: 5m
|
||||
tags:
|
||||
- key: service.name
|
||||
value: service
|
||||
queries:
|
||||
- name: Request rate
|
||||
query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))'
|
||||
- name: Error rate
|
||||
query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags,span_status="STATUS_CODE_ERROR"}[5m]))'
|
||||
@@ -1,10 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
global:
|
||||
scrape_interval: 1s
|
||||
evaluation_interval: 1s
|
||||
scrape_interval: 2s
|
||||
evaluation_interval: 5s
|
||||
external_labels:
|
||||
deployment_environment: lab
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: messenger
|
||||
@@ -9,6 +14,13 @@ scrape_configs:
|
||||
type: A
|
||||
port: 8080
|
||||
refresh_interval: 2s
|
||||
relabel_configs:
|
||||
- target_label: service_name
|
||||
replacement: messenger-api
|
||||
metric_relabel_configs:
|
||||
- source_labels: [__name__]
|
||||
regex: "go_.*|process_.*"
|
||||
action: keep
|
||||
|
||||
- job_name: nginx
|
||||
static_configs:
|
||||
@@ -32,12 +44,42 @@ scrape_configs:
|
||||
type: A
|
||||
port: 9091
|
||||
refresh_interval: 2s
|
||||
relabel_configs:
|
||||
- target_label: service_name
|
||||
replacement: messenger-worker
|
||||
metric_relabel_configs:
|
||||
- source_labels: [__name__]
|
||||
regex: "go_.*|process_.*"
|
||||
action: keep
|
||||
|
||||
- job_name: outbox-dispatcher
|
||||
static_configs:
|
||||
- targets: ["dispatcher:9091"]
|
||||
relabel_configs:
|
||||
- target_label: service_name
|
||||
replacement: messenger-dispatcher
|
||||
metric_relabel_configs:
|
||||
- source_labels: [__name__]
|
||||
regex: "go_.*|process_.*"
|
||||
action: keep
|
||||
|
||||
- job_name: minio
|
||||
metrics_path: /minio/v2/metrics/cluster
|
||||
static_configs:
|
||||
- targets: ["s3:9000"]
|
||||
|
||||
- job_name: otel-applications
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8889"]
|
||||
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8888"]
|
||||
|
||||
- job_name: tempo
|
||||
static_configs:
|
||||
- targets: ["tempo:3200"]
|
||||
|
||||
- job_name: loki
|
||||
static_configs:
|
||||
- targets: ["loki:3100"]
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
groups:
|
||||
- name: messenger-api-slo
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: MessengerAPIBusiness5xxHigh
|
||||
expr: |
|
||||
(
|
||||
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[2m])) or vector(0))
|
||||
/
|
||||
clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[2m])) or vector(0)), 0.000001)
|
||||
) > 0.05
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
service: messenger-api
|
||||
slo: api-availability
|
||||
annotations:
|
||||
summary: Business API 5xx ratio is above 5%
|
||||
description: More than 5% of business requests returned 5xx for one minute. Health and metrics routes are excluded.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
|
||||
|
||||
- alert: MessengerAPIHighErrorBudgetBurnFast
|
||||
expr: |
|
||||
(
|
||||
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[5m])) or vector(0))
|
||||
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[5m])) or vector(0)), 0.000001)
|
||||
) > 0.072
|
||||
and
|
||||
(
|
||||
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[1h])) or vector(0))
|
||||
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[1h])) or vector(0)), 0.000001)
|
||||
) > 0.072
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
service: messenger-api
|
||||
slo: api-availability
|
||||
annotations:
|
||||
summary: API availability error budget is burning fast
|
||||
description: The 99.5% API availability SLO burns above 14.4x in both 5m and 1h windows.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
|
||||
|
||||
- alert: MessengerAPIHighErrorBudgetBurnSlow
|
||||
expr: |
|
||||
(
|
||||
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[30m])) or vector(0))
|
||||
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[30m])) or vector(0)), 0.000001)
|
||||
) > 0.03
|
||||
and
|
||||
(
|
||||
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[6h])) or vector(0))
|
||||
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[6h])) or vector(0)), 0.000001)
|
||||
) > 0.03
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
service: messenger-api
|
||||
slo: api-availability
|
||||
annotations:
|
||||
summary: API availability error budget is burning steadily
|
||||
description: The 99.5% API availability SLO burns above 6x in both 30m and 6h windows.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
|
||||
|
||||
- name: messenger-attachment-slo
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: AttachmentReadyHighErrorBudgetBurnFast
|
||||
expr: |
|
||||
(
|
||||
clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0))
|
||||
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[5m])) or vector(0)),
|
||||
0
|
||||
)
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0))
|
||||
)
|
||||
/ clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0)),
|
||||
0.000001
|
||||
) > 0.144
|
||||
and
|
||||
(
|
||||
clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0))
|
||||
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h])) or vector(0)),
|
||||
0
|
||||
)
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0))
|
||||
)
|
||||
/ clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0)),
|
||||
0.000001
|
||||
) > 0.144
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
service: messenger-worker
|
||||
slo: attachment-ready
|
||||
annotations:
|
||||
summary: Attachment ready-time error budget is burning fast
|
||||
description: The 99% within 30s ready-time SLO burns above 14.4x in both 5m and 1h windows.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
||||
|
||||
- alert: AttachmentReadyHighErrorBudgetBurnSlow
|
||||
expr: |
|
||||
(
|
||||
clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0))
|
||||
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[30m])) or vector(0)),
|
||||
0
|
||||
)
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0))
|
||||
)
|
||||
/ clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0)),
|
||||
0.000001
|
||||
) > 0.06
|
||||
and
|
||||
(
|
||||
clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0))
|
||||
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[6h])) or vector(0)),
|
||||
0
|
||||
)
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0))
|
||||
)
|
||||
/ clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0)),
|
||||
0.000001
|
||||
) > 0.06
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
service: messenger-worker
|
||||
slo: attachment-ready
|
||||
annotations:
|
||||
summary: Attachment ready-time error budget is burning steadily
|
||||
description: The 99% within 30s ready-time SLO burns above 6x in both 30m and 6h windows.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
||||
|
||||
- alert: AttachmentWorkerUnavailable
|
||||
expr: (sum(up{job="attachment-worker"}) or vector(0)) < 1
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
service: messenger-worker
|
||||
annotations:
|
||||
summary: No attachment worker is available
|
||||
description: Prometheus has not observed a healthy worker target for 30 seconds.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
||||
|
||||
- alert: AttachmentQueueBacklogGrowing
|
||||
expr: |
|
||||
max(messenger_attachment_jobs_oldest_age_seconds{status="queued"}) > 30
|
||||
and max(deriv(messenger_attachment_jobs_oldest_age_seconds{status="queued"}[2m])) > 0
|
||||
for: 30s
|
||||
labels:
|
||||
severity: warning
|
||||
service: messenger-worker
|
||||
annotations:
|
||||
summary: Attachment queue backlog is growing
|
||||
description: The oldest queued attachment is older than 30 seconds and its age continues to increase.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
||||
|
||||
- alert: AttachmentRedisStreamLagHigh
|
||||
expr: max(redis_stream_group_lag{job="redis-broker",stream="attachment-jobs",group="attachment-workers"}) > 20
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
service: redis-broker
|
||||
annotations:
|
||||
summary: Redis Stream consumer lag is high
|
||||
description: The attachment worker group has more than 20 messages waiting for delivery.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
||||
@@ -0,0 +1,72 @@
|
||||
# API availability SLO
|
||||
|
||||
## Цель и сигналы
|
||||
|
||||
Объект контроля — бизнес-маршруты API за Nginx. Health endpoints исключены. Хорошими считаются ответы `2xx` и ожидаемые клиентские `4xx`; плохими — `5xx`. Цель — не менее 99,5% хороших ответов за скользящие 30 дней, error budget равен 0,5%.
|
||||
|
||||
Основные сигналы:
|
||||
|
||||
- `messenger_http_requests_total{method,route,status_class}` — число запросов;
|
||||
- `messenger_http_request_duration_seconds_bucket{method,route}` — распределение server latency;
|
||||
- `up{job="messenger"}` и `up{job="nginx"}` — доступность scrape targets;
|
||||
- `messenger_db_pool_connections{state="used"}` и `rate(messenger_db_pool_waited_total[5m])` — использование пула и частота ожиданий свободного соединения;
|
||||
- trace `messenger-api` — путь одного запроса по зависимостям;
|
||||
- JSON-логи `messenger-api` с `request_id`, `trace_id` и `span_id`.
|
||||
|
||||
`user_id`, `chat_id`, `message_id`, `attachment_id` и `request_id` не используются как labels метрик.
|
||||
|
||||
## Диагностика: metrics → trace → logs
|
||||
|
||||
1. В Grafana открыть dashboard **Messenger Lab 6 / API** и установить общий временной диапазон инцидента.
|
||||
2. Проверить error ratio, RPS, p95/p99 и `up`. Рост `5xx` при нормальном RPS подтверждает влияние на SLI; отсутствие трафика не считается успешностью.
|
||||
3. Сравнить DB pool used/waiting и состояние PostgreSQL. Рост ожидания пула указывает на зависимость, а не на CPU handler.
|
||||
4. В панели ошибок выбрать exemplar или перейти в Explore → Tempo. Если exemplar отсутствует, выполнить TraceQL:
|
||||
|
||||
```text
|
||||
{ resource.service.name = "messenger-api" && span.http.route != "/health/ready" && status = error }
|
||||
```
|
||||
|
||||
5. В trace найти первый error span. Проверить `http.route`, `http.request.method`, `http.response.status_code`, `db.system`, duration и resource attributes. Полный URL, токен и тело запроса в span отсутствуют.
|
||||
6. Скопировать `trace_id`, перейти по trace-to-logs или выполнить в Explore → Loki:
|
||||
|
||||
```text
|
||||
{service_name="messenger-api"} | json | trace_id="<trace_id>"
|
||||
```
|
||||
|
||||
7. Сверить `request_id`, событие, severity и error class. Текст сообщений, `Authorization` и пароли в логах отсутствуют.
|
||||
|
||||
## Проверка alert
|
||||
|
||||
Состояние правил видно в Grafana Alerting и в Prometheus:
|
||||
|
||||
```bash
|
||||
curl -s http://127.0.0.1:9090/api/v1/alerts
|
||||
```
|
||||
|
||||
Контролируемый отказ PostgreSQL:
|
||||
|
||||
```bash
|
||||
ENV_FILE=lab6.env.example scripts/incident-database.sh
|
||||
```
|
||||
|
||||
Сценарий сохраняет baseline, нагрузку, состояние деградации, `alert-firing.json`, восстановление и `alert-resolved.json` в `evidence/runtime/`. Trap запускает PostgreSQL и API даже при ошибке сценария. Volumes не удаляются.
|
||||
|
||||
Ожидаемая последовательность:
|
||||
|
||||
```text
|
||||
нормальный error ratio
|
||||
→ PostgreSQL остановлен
|
||||
→ 5xx растёт
|
||||
→ MessengerAPIBusiness5xxHigh и MessengerAPIHighErrorBudgetBurnFast firing
|
||||
→ PostgreSQL восстановлен
|
||||
→ успешные ответы возвращаются
|
||||
→ оперативный alert resolved; burn-rate хранит историю длинного окна
|
||||
```
|
||||
|
||||
## Действия
|
||||
|
||||
1. Если `up{job="nginx"}=0`, восстановить ingress и проверить его логи.
|
||||
2. Если API targets недоступны, восстановить реплики по одной и проверить readiness.
|
||||
3. Если API работает, а DB spans завершаются ошибкой или pool waiting растёт, восстановить PostgreSQL и не увеличивать число API-реплик до устранения причины.
|
||||
4. После восстановления проверить успешный smoke, отсутствие firing alerts и возвращение latency к baseline.
|
||||
5. Не удалять volumes и не сбрасывать очередь ради закрытия alert.
|
||||
@@ -0,0 +1,85 @@
|
||||
# Attachment pipeline SLO
|
||||
|
||||
## Цель и сигналы
|
||||
|
||||
Объект контроля — время от durable acceptance изображения до состояния `ready`. Цель — не менее 99% jobs завершаются `ready` не позднее 30 секунд за скользящие 30 дней. `failed` и завершение позже 30 секунд расходуют error budget.
|
||||
|
||||
Основные сигналы:
|
||||
|
||||
- `messenger_attachment_worker_jobs_total{job_type,result}` — исходы worker;
|
||||
- `messenger_attachment_worker_job_duration_seconds_bucket{job_type,result}` — время обработки;
|
||||
- `messenger_attachment_time_to_ready_seconds_bucket` — end-to-end время;
|
||||
- `messenger_attachment_jobs_count{status}` — состояния jobs в PostgreSQL;
|
||||
- `messenger_attachment_jobs_oldest_age_seconds{status}` — возраст старейшей job;
|
||||
- `messenger_attachment_worker_jobs_in_flight` и `messenger_attachment_worker_capacity` — насыщение worker;
|
||||
- `messenger_outbox_pending` — задержка до публикации;
|
||||
- Redis Stream lag и PEL — доставка и неподтверждённые сообщения.
|
||||
|
||||
Queue depth не заменяет processing latency: очередь может быть короткой, но содержать одну очень старую job.
|
||||
|
||||
## Диагностика: metrics → trace → logs
|
||||
|
||||
1. В Grafana открыть dashboard **Messenger Lab 6 / Attachment pipeline** и выбрать диапазон инцидента.
|
||||
2. Определить место задержки:
|
||||
- растёт `messenger_outbox_pending` — проблема до публикации в broker;
|
||||
- растут stream lag и `queued` — worker не успевает или недоступен;
|
||||
- растут PEL и `processing` — delivery получена, но не завершена;
|
||||
- растёт `failed` — постоянные ошибки или исчерпаны retry;
|
||||
- busy равен capacity — worker насыщен.
|
||||
3. В Explore → Tempo найти upload по route или worker consumer span:
|
||||
|
||||
```text
|
||||
{ resource.service.name = "messenger-api" && span.http.route = "/api/v1/chats/{chat_id}/attachments" }
|
||||
```
|
||||
|
||||
```text
|
||||
{ resource.service.name = "messenger-worker" && name = "attachment.process" }
|
||||
```
|
||||
|
||||
4. В trace пройти цепочку `HTTP upload → outbox publish → Redis consume → S3 variants → PostgreSQL ready`. Проверить producer/consumer связь и retry spans. Технический `attachment_id` допустим только как span/log attribute синтетического стенда, но не как metric label.
|
||||
5. Перейти из выбранного span в Loki или выполнить:
|
||||
|
||||
```text
|
||||
{service_name="messenger-worker"} | json | trace_id="<trace_id>"
|
||||
```
|
||||
|
||||
6. Сверить в логе `event`, `job_type`, `attempt`, `result`, `trace_id` и `span_id`; в span проверить `job.outcome`. Payload envelope, изображение и credentials не логируются.
|
||||
|
||||
## Проверка alert
|
||||
|
||||
Короткая деградация:
|
||||
|
||||
```bash
|
||||
ENV_FILE=lab6.env.example scripts/incident-worker.sh short
|
||||
```
|
||||
|
||||
Длительная деградация:
|
||||
|
||||
```bash
|
||||
ENV_FILE=lab6.env.example scripts/incident-worker.sh long
|
||||
```
|
||||
|
||||
Ожидаемая последовательность:
|
||||
|
||||
```text
|
||||
worker up, backlog около нуля
|
||||
→ worker остановлен
|
||||
→ queued и oldest age растут
|
||||
→ short: AttachmentWorkerUnavailable firing
|
||||
→ long: AttachmentQueueBacklogGrowing firing
|
||||
→ worker запущен
|
||||
→ backlog дренируется
|
||||
→ все принятые attachments достигают ready
|
||||
→ alert resolved
|
||||
```
|
||||
|
||||
Каждый сценарий восстанавливает worker через trap и сохраняет evidence без session token и raw payload.
|
||||
|
||||
## Действия
|
||||
|
||||
1. Если worker `up=0`, восстановить worker и наблюдать скорость дренирования.
|
||||
2. Если worker доступен, но busy равно capacity, проверить S3/DB spans до масштабирования concurrency.
|
||||
3. Если растёт outbox, проверить dispatcher и PostgreSQL lease; Redis worker здесь ещё не причина.
|
||||
4. Если растёт PEL, проверить heartbeat, lease expiry и reclaim.
|
||||
5. Для `failed` определить permanent/transient error по логу и trace. Replay выполнять только после устранения причины.
|
||||
6. После восстановления проверить backlog, oldest age, `failed`, firing alerts и terminal state каждого синтетического attachment.
|
||||
Reference in New Issue
Block a user