ops: add observability stack and SLO alerts

This commit is contained in:
2026-11-01 17:06:35 +03:00
parent 934dfefee7
commit 4bf383bd51
15 changed files with 1341 additions and 32 deletions
+2 -1
View File
@@ -20,8 +20,9 @@ ENV SWAGGER_JSON=/spec/openapi.json
FROM prom/prometheus:v3.13.3 AS prometheus
COPY monitoring/prometheus.yml /etc/prometheus/prometheus.yml
COPY monitoring/rules /etc/prometheus/rules
FROM grafana/grafana:13.1.6 AS grafana
FROM grafana/grafana:13.2.4 AS grafana
COPY monitoring/grafana/provisioning /etc/grafana/provisioning
COPY monitoring/grafana/dashboards /etc/grafana/dashboards
+94 -9
View File
@@ -4,6 +4,11 @@ x-messenger-config-volume: &messenger-config-volume
target: /etc/messenger/messenger.yaml
read_only: true
x-telemetry-environment: &telemetry-environment
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT}
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL}
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${DEPLOYMENT_ENVIRONMENT},service.version=${SERVICE_VERSION}
services:
postgres:
image: postgres:17.6-alpine3.22
@@ -118,7 +123,7 @@ services:
networks: [data]
migrate:
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:lab6}
command: ["/app/messenger", "migrate"]
environment:
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
@@ -137,8 +142,10 @@ services:
api:
build: .
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:lab6}
environment:
<<: *telemetry-environment
OTEL_SERVICE_NAME: messenger-api
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
APP_HOST: ${APP_HOST}
APP_PORT: ${APP_PORT}
@@ -176,9 +183,11 @@ services:
networks: [application, data]
dispatcher:
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:lab6}
command: ["/app/messenger", "dispatch"]
environment:
<<: *telemetry-environment
OTEL_SERVICE_NAME: messenger-dispatcher
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
DATABASE_URL: ${DATABASE_URL}
BROKER_URL: ${BROKER_URL}
@@ -202,9 +211,11 @@ services:
networks: [data, monitoring]
worker:
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:lab6}
command: ["/app/messenger", "worker"]
environment:
<<: *telemetry-environment
OTEL_SERVICE_NAME: messenger-worker
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
DATABASE_URL: ${DATABASE_URL}
S3_ENDPOINT: ${S3_ENDPOINT}
@@ -239,7 +250,7 @@ services:
build:
context: .
target: loadbalancer
image: ${NGINX_IMAGE:-mtusi-nginx:lab5}
image: ${NGINX_IMAGE:-mtusi-nginx:lab6}
ports:
- "${API_BIND_ADDRESS}:8080"
depends_on:
@@ -259,7 +270,7 @@ services:
build:
context: .
target: swagger
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab5}
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab6}
ports:
- "${SWAGGER_BIND_ADDRESS}:8080"
depends_on:
@@ -289,7 +300,7 @@ services:
lab5-experiment:
profiles: ["experiments"]
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:lab6}
command: ["/app/lab5experiment"]
environment:
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
@@ -375,11 +386,12 @@ services:
build:
context: .
target: prometheus
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab5}
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab6}
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=7d
- --web.enable-remote-write-receiver
volumes:
- prometheus_data:/prometheus
ports:
@@ -398,6 +410,9 @@ services:
- dispatcher
- worker
- s3
- otel-collector
- loki
- tempo
cpus: ${PROMETHEUS_CPUS}
mem_limit: ${PROMETHEUS_MEMORY}
restart: unless-stopped
@@ -407,28 +422,98 @@ services:
build:
context: .
target: grafana
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab5}
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab6}
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
GF_USERS_ALLOW_SIGN_UP: "false"
GF_EXPLORE_ENABLED: "true"
volumes:
- grafana_data:/var/lib/grafana
ports:
- "${GRAFANA_BIND_ADDRESS}:3000"
depends_on:
- prometheus
- loki
- tempo
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health || exit 1"]
interval: 5s
timeout: 3s
retries: 20
cpus: ${GRAFANA_CPUS}
mem_limit: ${GRAFANA_MEMORY}
restart: unless-stopped
networks: [edge, monitoring]
loki:
image: grafana/loki:3.7.9
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- loki_data:/loki
ports:
- "${LOKI_BIND_ADDRESS}:3100"
healthcheck:
test: ["CMD", "/usr/bin/loki", "-version"]
interval: 10s
timeout: 3s
retries: 10
cpus: ${LOKI_CPUS}
mem_limit: ${LOKI_MEMORY}
restart: unless-stopped
networks: [edge, monitoring]
tempo:
image: grafana/tempo:2.10.10
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempo_data:/var/tempo
ports:
- "${TEMPO_BIND_ADDRESS}:3200"
healthcheck:
test: ["CMD", "/tempo", "-version"]
interval: 10s
timeout: 3s
retries: 10
cpus: ${TEMPO_CPUS}
mem_limit: ${TEMPO_MEMORY}
restart: unless-stopped
networks: [edge, monitoring]
otel-collector:
image: otel/opentelemetry-collector-contrib:0.160.0
command: ["--config=/etc/otelcol-contrib/collector.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro
expose:
- "4317"
- "4318"
- "8888"
- "8889"
- "13133"
depends_on:
- loki
- tempo
healthcheck:
test: ["CMD", "/otelcol-contrib", "components"]
interval: 10s
timeout: 3s
retries: 10
cpus: ${OTEL_COLLECTOR_CPUS}
mem_limit: ${OTEL_COLLECTOR_MEMORY}
restart: unless-stopped
networks: [application, data, monitoring]
volumes:
postgres_data:
valkey_data:
redis_broker_data:
prometheus_data:
grafana_data:
loki_data:
tempo_data:
s3_data:
networks:
+48
View File
@@ -0,0 +1,48 @@
auth_enabled: false
server:
http_listen_address: 0.0.0.0
http_listen_port: 3100
grpc_listen_address: 0.0.0.0
grpc_listen_port: 9096
common:
path_prefix: /loki
replication_factor: 1
ring:
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 168h
volume_enabled: true
otlp_config:
resource_attributes:
attributes_config:
- action: index_label
attributes:
- service.name
- deployment.environment.name
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+73
View File
@@ -0,0 +1,73 @@
extensions:
health_check:
endpoint: 0.0.0.0:13133
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 192
spike_limit_mib: 48
batch:
send_batch_size: 1024
send_batch_max_size: 2048
timeout: 2s
exporters:
prometheus:
endpoint: 0.0.0.0:8889
resource_constant_labels:
included:
- service.name
- service.version
- deployment.environment.name
otlp_grpc/tempo:
endpoint: tempo:4317
tls:
insecure: true
sending_queue:
enabled: true
num_consumers: 2
queue_size: 512
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 30s
otlp_http/loki:
endpoint: http://loki:3100/otlp
sending_queue:
enabled: true
num_consumers: 2
queue_size: 512
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 30s
service:
extensions: [health_check]
telemetry:
logs:
level: info
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp_http/loki]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp_grpc/tempo]
+48
View File
@@ -0,0 +1,48 @@
server:
http_listen_address: 0.0.0.0
http_listen_port: 3200
grpc_listen_address: 0.0.0.0
grpc_listen_port: 9095
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
ingester:
max_block_duration: 5m
compactor:
compaction:
block_retention: 168h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/traces
metrics_generator:
registry:
external_labels:
source: tempo
deployment_environment: lab
storage:
path: /var/tempo/generator/wal
remote_write:
- url: http://prometheus:9090/api/v1/write
send_exemplars: true
overrides:
defaults:
metrics_generator:
processors: [service-graphs, span-metrics]
usage_report:
reporting_enabled: false
+19 -2
View File
@@ -1,3 +1,6 @@
# Изолированное имя Compose-проекта и его volumes/networks.
COMPOSE_PROJECT_NAME=mtusi2026lab6
# Адрес и внутренний порт HTTP-сервера.
APP_HOST=0.0.0.0
APP_PORT=8080
@@ -8,6 +11,14 @@ API_BIND_ADDRESS=127.0.0.1:8080
SWAGGER_BIND_ADDRESS=127.0.0.1:8081
PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090
GRAFANA_BIND_ADDRESS=127.0.0.1:3000
LOKI_BIND_ADDRESS=127.0.0.1:3100
TEMPO_BIND_ADDRESS=127.0.0.1:3200
# Ресурсные атрибуты и внутренний OTLP endpoint приложений.
DEPLOYMENT_ENVIRONMENT=lab
SERVICE_VERSION=lab6
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
OTEL_EXPORTER_OTLP_PROTOCOL=grpc
# Число API-реплик за Nginx.
API_REPLICAS=2
@@ -55,8 +66,14 @@ EXPORTER_CPUS=0.25
EXPORTER_MEMORY=128m
PROMETHEUS_CPUS=1
PROMETHEUS_MEMORY=512m
GRAFANA_CPUS=0.5
GRAFANA_MEMORY=256m
GRAFANA_CPUS=1
GRAFANA_MEMORY=512m
OTEL_COLLECTOR_CPUS=0.5
OTEL_COLLECTOR_MEMORY=256m
LOKI_CPUS=1
LOKI_MEMORY=512m
TEMPO_CPUS=1
TEMPO_MEMORY=512m
POSTGRES_CPUS=2
POSTGRES_MEMORY=1g
VALKEY_CPUS=1
@@ -67,7 +67,7 @@
},
{
"type": "timeseries",
"title": "API latency p95 by route",
"title": "API latency p50 / p95 / p99 by route",
"id": 3,
"gridPos": {
"h": 8,
@@ -77,9 +77,19 @@
},
"targets": [
{
"expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
"legendFormat": "{{method}} {{route}}",
"expr": "histogram_quantile(0.50, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
"legendFormat": "p50 {{method}} {{route}}",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
"legendFormat": "p95 {{method}} {{route}}",
"refId": "B"
},
{
"expr": "histogram_quantile(0.99, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
"legendFormat": "p99 {{method}} {{route}}",
"refId": "C"
}
],
"fieldConfig": {
@@ -895,14 +905,563 @@
}
}
]
},
{
"type": "row",
"title": "SLO and alerts",
"id": 105,
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 109
},
"panels": [
{
"type": "timeseries",
"title": "Business API availability",
"id": 31,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 110
},
"targets": [
{
"expr": "1 - (sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001)",
"legendFormat": "availability",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "API error-budget burn rate (99.5% SLO)",
"id": 32,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 110
},
"targets": [
{
"expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001) / 0.005",
"legendFormat": "5m burn",
"refId": "A"
},
{
"expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[1h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[1h])) or vector(0)), 0.000001) / 0.005",
"legendFormat": "1h burn",
"refId": "B"
},
{
"expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[6h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[6h])) or vector(0)), 0.000001) / 0.005",
"legendFormat": "6h burn",
"refId": "C"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "Attachments ready within 30 seconds",
"id": 33,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 118
},
"targets": [
{
"expr": "1 - (clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001)",
"legendFormat": "ready in 30s",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "Attachment ready-time error-budget burn",
"id": 34,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 118
},
"targets": [
{
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001) / 0.01",
"legendFormat": "5m burn",
"refId": "A"
},
{
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[1h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0)), 0.000001) / 0.01",
"legendFormat": "1h burn",
"refId": "B"
},
{
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[6h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0)), 0.000001) / 0.01",
"legendFormat": "6h burn",
"refId": "C"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "SLO alert state",
"id": 35,
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 126
},
"targets": [
{
"expr": "max by (alertname, alertstate) (ALERTS{alertname=~\"MessengerAPIHighErrorBudgetBurn.*|AttachmentReadyHighErrorBudgetBurn.*|AttachmentQueueBacklogGrowing|AttachmentWorkerUnavailable\"})",
"legendFormat": "{{alertname}} {{alertstate}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
}
]
},
{
"type": "row",
"title": "Runtime resources",
"id": 107,
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 134
},
"panels": [
{
"type": "timeseries",
"title": "Process CPU by service",
"id": 38,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 135
},
"targets": [
{
"expr": "sum by (service_name) (rate(process_cpu_seconds_total[1m]))",
"legendFormat": "{{service_name}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "cores"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "Go heap by service",
"id": 39,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 135
},
"targets": [
{
"expr": "sum by (service_name) (go_memstats_heap_alloc_bytes)",
"legendFormat": "{{service_name}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "Database pool used and wait rate",
"id": 40,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 143
},
"targets": [
{
"expr": "sum by (service_name) (messenger_db_pool_connections{state=\"used\"})",
"legendFormat": "{{service_name}} used",
"refId": "A"
},
{
"expr": "sum by (service_name) (rate(messenger_db_pool_waited_total[5m]))",
"legendFormat": "{{service_name}} waits/s",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "Worker saturation",
"id": 41,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 143
},
"targets": [
{
"expr": "sum(messenger_attachment_worker_jobs_in_flight)",
"legendFormat": "busy",
"refId": "A"
},
{
"expr": "sum(messenger_attachment_worker_capacity)",
"legendFormat": "capacity",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
}
]
},
{
"type": "row",
"title": "Logs and traces",
"id": 106,
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 151
},
"panels": [
{
"type": "logs",
"title": "Correlated application logs",
"id": 36,
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 152
},
"datasource": {
"type": "loki",
"uid": "loki"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "code",
"expr": "{service_name=~\"messenger-.+\"} | json",
"queryType": "range",
"refId": "A"
}
],
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
}
},
{
"type": "traces",
"title": "Recent application traces (TraceQL)",
"id": 37,
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 152
},
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"targets": [
{
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"query": "{ resource.service.name =~ \"messenger-.*\" }",
"queryType": "traceql",
"refId": "A",
"limit": 50,
"tableType": "traces"
}
],
"options": {
"showSpanFilters": true
}
},
{
"type": "timeseries",
"title": "Trace-derived service call rate",
"id": 42,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 162
},
"targets": [
{
"expr": "sum by (service, span_name) (rate(traces_spanmetrics_calls_total[5m]))",
"legendFormat": "{{service}} {{span_name}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"type": "timeseries",
"title": "Collector delivery failures and queue pressure",
"id": 43,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 162
},
"targets": [
{
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans_total[5m]))",
"legendFormat": "{{exporter}} spans failed/s",
"refId": "A"
},
{
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records_total[5m]))",
"legendFormat": "{{exporter}} logs failed/s",
"refId": "B"
},
{
"expr": "max by (exporter) (otelcol_exporter_queue_size)",
"legendFormat": "{{exporter}} queue",
"refId": "C"
},
{
"expr": "max by (exporter) (otelcol_exporter_queue_capacity)",
"legendFormat": "{{exporter}} capacity",
"refId": "D"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
]
}
],
"refresh": "5s",
"schemaVersion": 41,
"tags": [
"highload",
"lab5",
"pipeline"
"lab6",
"observability",
"slo"
],
"templating": {
"list": []
@@ -912,7 +1471,42 @@
"to": "now"
},
"timezone": "browser",
"title": "Messenger lab5 overview",
"uid": "messenger-lab5",
"version": 3
"title": "Messenger lab6 observability",
"uid": "messenger-lab6",
"version": 1,
"links": [
{
"asDropdown": false,
"icon": "external link",
"includeVars": true,
"keepTime": true,
"tags": [],
"targetBlank": false,
"title": "Explore logs and traces",
"type": "link",
"url": "/explore"
},
{
"asDropdown": false,
"icon": "external link",
"includeVars": false,
"keepTime": false,
"tags": [],
"targetBlank": true,
"title": "API runbook",
"type": "link",
"url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md"
},
{
"asDropdown": false,
"icon": "external link",
"includeVars": false,
"keepTime": false,
"tags": [],
"targetBlank": true,
"title": "Pipeline runbook",
"type": "link",
"url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md"
}
]
}
@@ -0,0 +1 @@
@@ -0,0 +1,74 @@
apiVersion: 1
deleteDatasources:
- name: Prometheus
orgId: 1
- name: Loki
orgId: 1
- name: Tempo
orgId: 1
datasources:
- name: Prometheus
type: prometheus
uid: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
jsonData:
httpMethod: POST
exemplarTraceIdDestinations:
- name: trace_id
datasourceUid: tempo
- name: Loki
type: loki
uid: loki
access: proxy
url: http://loki:3100
editable: false
jsonData:
maxLines: 1000
derivedFields:
- name: TraceID
matcherRegex: 'trace_id[=:" ]+([0-9a-f]{32})'
datasourceUid: tempo
url: '$${__value.raw}'
- name: Tempo
type: tempo
uid: tempo
access: proxy
url: http://tempo:3200
editable: false
jsonData:
nodeGraph:
enabled: true
search:
hide: false
serviceMap:
datasourceUid: prometheus
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: -5m
spanEndTimeShift: 5m
filterByTraceID: true
filterBySpanID: false
customQuery: true
query: '{service_name="$${__span.tags["service.name"]}"} | trace_id = "$${__trace.traceId}"'
tags:
- key: service.name
value: service_name
tracesToMetrics:
datasourceUid: prometheus
spanStartTimeShift: -5m
spanEndTimeShift: 5m
tags:
- key: service.name
value: service
queries:
- name: Request rate
query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))'
- name: Error rate
query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags,span_status="STATUS_CODE_ERROR"}[5m]))'
@@ -1,10 +0,0 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
uid: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
@@ -0,0 +1 @@
+44 -2
View File
@@ -1,6 +1,11 @@
global:
scrape_interval: 1s
evaluation_interval: 1s
scrape_interval: 2s
evaluation_interval: 5s
external_labels:
deployment_environment: lab
rule_files:
- /etc/prometheus/rules/*.yml
scrape_configs:
- job_name: messenger
@@ -9,6 +14,13 @@ scrape_configs:
type: A
port: 8080
refresh_interval: 2s
relabel_configs:
- target_label: service_name
replacement: messenger-api
metric_relabel_configs:
- source_labels: [__name__]
regex: "go_.*|process_.*"
action: keep
- job_name: nginx
static_configs:
@@ -32,12 +44,42 @@ scrape_configs:
type: A
port: 9091
refresh_interval: 2s
relabel_configs:
- target_label: service_name
replacement: messenger-worker
metric_relabel_configs:
- source_labels: [__name__]
regex: "go_.*|process_.*"
action: keep
- job_name: outbox-dispatcher
static_configs:
- targets: ["dispatcher:9091"]
relabel_configs:
- target_label: service_name
replacement: messenger-dispatcher
metric_relabel_configs:
- source_labels: [__name__]
regex: "go_.*|process_.*"
action: keep
- job_name: minio
metrics_path: /minio/v2/metrics/cluster
static_configs:
- targets: ["s3:9000"]
- job_name: otel-applications
static_configs:
- targets: ["otel-collector:8889"]
- job_name: otel-collector
static_configs:
- targets: ["otel-collector:8888"]
- job_name: tempo
static_configs:
- targets: ["tempo:3200"]
- job_name: loki
static_configs:
- targets: ["loki:3100"]
+178
View File
@@ -0,0 +1,178 @@
groups:
- name: messenger-api-slo
interval: 5s
rules:
- alert: MessengerAPIBusiness5xxHigh
expr: |
(
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[2m])) or vector(0))
/
clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[2m])) or vector(0)), 0.000001)
) > 0.05
for: 1m
labels:
severity: warning
service: messenger-api
slo: api-availability
annotations:
summary: Business API 5xx ratio is above 5%
description: More than 5% of business requests returned 5xx for one minute. Health and metrics routes are excluded.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
- alert: MessengerAPIHighErrorBudgetBurnFast
expr: |
(
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[5m])) or vector(0))
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[5m])) or vector(0)), 0.000001)
) > 0.072
and
(
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[1h])) or vector(0))
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[1h])) or vector(0)), 0.000001)
) > 0.072
for: 30s
labels:
severity: critical
service: messenger-api
slo: api-availability
annotations:
summary: API availability error budget is burning fast
description: The 99.5% API availability SLO burns above 14.4x in both 5m and 1h windows.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
- alert: MessengerAPIHighErrorBudgetBurnSlow
expr: |
(
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[30m])) or vector(0))
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[30m])) or vector(0)), 0.000001)
) > 0.03
and
(
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[6h])) or vector(0))
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[6h])) or vector(0)), 0.000001)
) > 0.03
for: 1m
labels:
severity: warning
service: messenger-api
slo: api-availability
annotations:
summary: API availability error budget is burning steadily
description: The 99.5% API availability SLO burns above 6x in both 30m and 6h windows.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
- name: messenger-attachment-slo
interval: 5s
rules:
- alert: AttachmentReadyHighErrorBudgetBurnFast
expr: |
(
clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0))
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[5m])) or vector(0)),
0
)
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0))
)
/ clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0))
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0)),
0.000001
) > 0.144
and
(
clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0))
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h])) or vector(0)),
0
)
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0))
)
/ clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0))
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0)),
0.000001
) > 0.144
for: 30s
labels:
severity: critical
service: messenger-worker
slo: attachment-ready
annotations:
summary: Attachment ready-time error budget is burning fast
description: The 99% within 30s ready-time SLO burns above 14.4x in both 5m and 1h windows.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
- alert: AttachmentReadyHighErrorBudgetBurnSlow
expr: |
(
clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0))
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[30m])) or vector(0)),
0
)
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0))
)
/ clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0))
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0)),
0.000001
) > 0.06
and
(
clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0))
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[6h])) or vector(0)),
0
)
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0))
)
/ clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0))
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0)),
0.000001
) > 0.06
for: 1m
labels:
severity: warning
service: messenger-worker
slo: attachment-ready
annotations:
summary: Attachment ready-time error budget is burning steadily
description: The 99% within 30s ready-time SLO burns above 6x in both 30m and 6h windows.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
- alert: AttachmentWorkerUnavailable
expr: (sum(up{job="attachment-worker"}) or vector(0)) < 1
for: 30s
labels:
severity: critical
service: messenger-worker
annotations:
summary: No attachment worker is available
description: Prometheus has not observed a healthy worker target for 30 seconds.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
- alert: AttachmentQueueBacklogGrowing
expr: |
max(messenger_attachment_jobs_oldest_age_seconds{status="queued"}) > 30
and max(deriv(messenger_attachment_jobs_oldest_age_seconds{status="queued"}[2m])) > 0
for: 30s
labels:
severity: warning
service: messenger-worker
annotations:
summary: Attachment queue backlog is growing
description: The oldest queued attachment is older than 30 seconds and its age continues to increase.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
- alert: AttachmentRedisStreamLagHigh
expr: max(redis_stream_group_lag{job="redis-broker",stream="attachment-jobs",group="attachment-workers"}) > 20
for: 1m
labels:
severity: warning
service: redis-broker
annotations:
summary: Redis Stream consumer lag is high
description: The attachment worker group has more than 20 messages waiting for delivery.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
+72
View File
@@ -0,0 +1,72 @@
# API availability SLO
## Цель и сигналы
Объект контроля — бизнес-маршруты API за Nginx. Health endpoints исключены. Хорошими считаются ответы `2xx` и ожидаемые клиентские `4xx`; плохими — `5xx`. Цель — не менее 99,5% хороших ответов за скользящие 30 дней, error budget равен 0,5%.
Основные сигналы:
- `messenger_http_requests_total{method,route,status_class}` — число запросов;
- `messenger_http_request_duration_seconds_bucket{method,route}` — распределение server latency;
- `up{job="messenger"}` и `up{job="nginx"}` — доступность scrape targets;
- `messenger_db_pool_connections{state="used"}` и `rate(messenger_db_pool_waited_total[5m])` — использование пула и частота ожиданий свободного соединения;
- trace `messenger-api` — путь одного запроса по зависимостям;
- JSON-логи `messenger-api` с `request_id`, `trace_id` и `span_id`.
`user_id`, `chat_id`, `message_id`, `attachment_id` и `request_id` не используются как labels метрик.
## Диагностика: metrics → trace → logs
1. В Grafana открыть dashboard **Messenger Lab 6 / API** и установить общий временной диапазон инцидента.
2. Проверить error ratio, RPS, p95/p99 и `up`. Рост `5xx` при нормальном RPS подтверждает влияние на SLI; отсутствие трафика не считается успешностью.
3. Сравнить DB pool used/waiting и состояние PostgreSQL. Рост ожидания пула указывает на зависимость, а не на CPU handler.
4. В панели ошибок выбрать exemplar или перейти в Explore → Tempo. Если exemplar отсутствует, выполнить TraceQL:
```text
{ resource.service.name = "messenger-api" && span.http.route != "/health/ready" && status = error }
```
5. В trace найти первый error span. Проверить `http.route`, `http.request.method`, `http.response.status_code`, `db.system`, duration и resource attributes. Полный URL, токен и тело запроса в span отсутствуют.
6. Скопировать `trace_id`, перейти по trace-to-logs или выполнить в Explore → Loki:
```text
{service_name="messenger-api"} | json | trace_id="<trace_id>"
```
7. Сверить `request_id`, событие, severity и error class. Текст сообщений, `Authorization` и пароли в логах отсутствуют.
## Проверка alert
Состояние правил видно в Grafana Alerting и в Prometheus:
```bash
curl -s http://127.0.0.1:9090/api/v1/alerts
```
Контролируемый отказ PostgreSQL:
```bash
ENV_FILE=lab6.env.example scripts/incident-database.sh
```
Сценарий сохраняет baseline, нагрузку, состояние деградации, `alert-firing.json`, восстановление и `alert-resolved.json` в `evidence/runtime/`. Trap запускает PostgreSQL и API даже при ошибке сценария. Volumes не удаляются.
Ожидаемая последовательность:
```text
нормальный error ratio
→ PostgreSQL остановлен
→ 5xx растёт
→ MessengerAPIBusiness5xxHigh и MessengerAPIHighErrorBudgetBurnFast firing
→ PostgreSQL восстановлен
→ успешные ответы возвращаются
→ оперативный alert resolved; burn-rate хранит историю длинного окна
```
## Действия
1. Если `up{job="nginx"}=0`, восстановить ingress и проверить его логи.
2. Если API targets недоступны, восстановить реплики по одной и проверить readiness.
3. Если API работает, а DB spans завершаются ошибкой или pool waiting растёт, восстановить PostgreSQL и не увеличивать число API-реплик до устранения причины.
4. После восстановления проверить успешный smoke, отсутствие firing alerts и возвращение latency к baseline.
5. Не удалять volumes и не сбрасывать очередь ради закрытия alert.
@@ -0,0 +1,85 @@
# Attachment pipeline SLO
## Цель и сигналы
Объект контроля — время от durable acceptance изображения до состояния `ready`. Цель — не менее 99% jobs завершаются `ready` не позднее 30 секунд за скользящие 30 дней. `failed` и завершение позже 30 секунд расходуют error budget.
Основные сигналы:
- `messenger_attachment_worker_jobs_total{job_type,result}` — исходы worker;
- `messenger_attachment_worker_job_duration_seconds_bucket{job_type,result}` — время обработки;
- `messenger_attachment_time_to_ready_seconds_bucket` — end-to-end время;
- `messenger_attachment_jobs_count{status}` — состояния jobs в PostgreSQL;
- `messenger_attachment_jobs_oldest_age_seconds{status}` — возраст старейшей job;
- `messenger_attachment_worker_jobs_in_flight` и `messenger_attachment_worker_capacity` — насыщение worker;
- `messenger_outbox_pending` — задержка до публикации;
- Redis Stream lag и PEL — доставка и неподтверждённые сообщения.
Queue depth не заменяет processing latency: очередь может быть короткой, но содержать одну очень старую job.
## Диагностика: metrics → trace → logs
1. В Grafana открыть dashboard **Messenger Lab 6 / Attachment pipeline** и выбрать диапазон инцидента.
2. Определить место задержки:
- растёт `messenger_outbox_pending` — проблема до публикации в broker;
- растут stream lag и `queued` — worker не успевает или недоступен;
- растут PEL и `processing` — delivery получена, но не завершена;
- растёт `failed` — постоянные ошибки или исчерпаны retry;
- busy равен capacity — worker насыщен.
3. В Explore → Tempo найти upload по route или worker consumer span:
```text
{ resource.service.name = "messenger-api" && span.http.route = "/api/v1/chats/{chat_id}/attachments" }
```
```text
{ resource.service.name = "messenger-worker" && name = "attachment.process" }
```
4. В trace пройти цепочку `HTTP upload → outbox publish → Redis consume → S3 variants → PostgreSQL ready`. Проверить producer/consumer связь и retry spans. Технический `attachment_id` допустим только как span/log attribute синтетического стенда, но не как metric label.
5. Перейти из выбранного span в Loki или выполнить:
```text
{service_name="messenger-worker"} | json | trace_id="<trace_id>"
```
6. Сверить в логе `event`, `job_type`, `attempt`, `result`, `trace_id` и `span_id`; в span проверить `job.outcome`. Payload envelope, изображение и credentials не логируются.
## Проверка alert
Короткая деградация:
```bash
ENV_FILE=lab6.env.example scripts/incident-worker.sh short
```
Длительная деградация:
```bash
ENV_FILE=lab6.env.example scripts/incident-worker.sh long
```
Ожидаемая последовательность:
```text
worker up, backlog около нуля
→ worker остановлен
→ queued и oldest age растут
→ short: AttachmentWorkerUnavailable firing
→ long: AttachmentQueueBacklogGrowing firing
→ worker запущен
→ backlog дренируется
→ все принятые attachments достигают ready
→ alert resolved
```
Каждый сценарий восстанавливает worker через trap и сохраняет evidence без session token и raw payload.
## Действия
1. Если worker `up=0`, восстановить worker и наблюдать скорость дренирования.
2. Если worker доступен, но busy равно capacity, проверить S3/DB spans до масштабирования concurrency.
3. Если растёт outbox, проверить dispatcher и PostgreSQL lease; Redis worker здесь ещё не причина.
4. Если растёт PEL, проверить heartbeat, lease expiry и reclaim.
5. Для `failed` определить permanent/transient error по логу и trace. Replay выполнять только после устранения причины.
6. После восстановления проверить backlog, oldest age, `failed`, firing alerts и terminal state каждого синтетического attachment.