forked from mtusi2026/mtusi2026lab6
ops: add observability stack and SLO alerts
This commit is contained in:
+2
-1
@@ -20,8 +20,9 @@ ENV SWAGGER_JSON=/spec/openapi.json
|
||||
FROM prom/prometheus:v3.13.3 AS prometheus
|
||||
|
||||
COPY monitoring/prometheus.yml /etc/prometheus/prometheus.yml
|
||||
COPY monitoring/rules /etc/prometheus/rules
|
||||
|
||||
FROM grafana/grafana:13.1.6 AS grafana
|
||||
FROM grafana/grafana:13.2.4 AS grafana
|
||||
|
||||
COPY monitoring/grafana/provisioning /etc/grafana/provisioning
|
||||
COPY monitoring/grafana/dashboards /etc/grafana/dashboards
|
||||
|
||||
+94
-9
@@ -4,6 +4,11 @@ x-messenger-config-volume: &messenger-config-volume
|
||||
target: /etc/messenger/messenger.yaml
|
||||
read_only: true
|
||||
|
||||
x-telemetry-environment: &telemetry-environment
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT}
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL}
|
||||
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${DEPLOYMENT_ENVIRONMENT},service.version=${SERVICE_VERSION}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.6-alpine3.22
|
||||
@@ -118,7 +123,7 @@ services:
|
||||
networks: [data]
|
||||
|
||||
migrate:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
command: ["/app/messenger", "migrate"]
|
||||
environment:
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
@@ -137,8 +142,10 @@ services:
|
||||
|
||||
api:
|
||||
build: .
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
environment:
|
||||
<<: *telemetry-environment
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
APP_HOST: ${APP_HOST}
|
||||
APP_PORT: ${APP_PORT}
|
||||
@@ -176,9 +183,11 @@ services:
|
||||
networks: [application, data]
|
||||
|
||||
dispatcher:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
command: ["/app/messenger", "dispatch"]
|
||||
environment:
|
||||
<<: *telemetry-environment
|
||||
OTEL_SERVICE_NAME: messenger-dispatcher
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
@@ -202,9 +211,11 @@ services:
|
||||
networks: [data, monitoring]
|
||||
|
||||
worker:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
command: ["/app/messenger", "worker"]
|
||||
environment:
|
||||
<<: *telemetry-environment
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT}
|
||||
@@ -239,7 +250,7 @@ services:
|
||||
build:
|
||||
context: .
|
||||
target: loadbalancer
|
||||
image: ${NGINX_IMAGE:-mtusi-nginx:lab5}
|
||||
image: ${NGINX_IMAGE:-mtusi-nginx:lab6}
|
||||
ports:
|
||||
- "${API_BIND_ADDRESS}:8080"
|
||||
depends_on:
|
||||
@@ -259,7 +270,7 @@ services:
|
||||
build:
|
||||
context: .
|
||||
target: swagger
|
||||
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab5}
|
||||
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab6}
|
||||
ports:
|
||||
- "${SWAGGER_BIND_ADDRESS}:8080"
|
||||
depends_on:
|
||||
@@ -289,7 +300,7 @@ services:
|
||||
|
||||
lab5-experiment:
|
||||
profiles: ["experiments"]
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
command: ["/app/lab5experiment"]
|
||||
environment:
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
@@ -375,11 +386,12 @@ services:
|
||||
build:
|
||||
context: .
|
||||
target: prometheus
|
||||
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab5}
|
||||
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab6}
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=7d
|
||||
- --web.enable-remote-write-receiver
|
||||
volumes:
|
||||
- prometheus_data:/prometheus
|
||||
ports:
|
||||
@@ -398,6 +410,9 @@ services:
|
||||
- dispatcher
|
||||
- worker
|
||||
- s3
|
||||
- otel-collector
|
||||
- loki
|
||||
- tempo
|
||||
cpus: ${PROMETHEUS_CPUS}
|
||||
mem_limit: ${PROMETHEUS_MEMORY}
|
||||
restart: unless-stopped
|
||||
@@ -407,28 +422,98 @@ services:
|
||||
build:
|
||||
context: .
|
||||
target: grafana
|
||||
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab5}
|
||||
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab6}
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_EXPLORE_ENABLED: "true"
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
ports:
|
||||
- "${GRAFANA_BIND_ADDRESS}:3000"
|
||||
depends_on:
|
||||
- prometheus
|
||||
- loki
|
||||
- tempo
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
cpus: ${GRAFANA_CPUS}
|
||||
mem_limit: ${GRAFANA_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [edge, monitoring]
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.7.9
|
||||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- loki_data:/loki
|
||||
ports:
|
||||
- "${LOKI_BIND_ADDRESS}:3100"
|
||||
healthcheck:
|
||||
test: ["CMD", "/usr/bin/loki", "-version"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
cpus: ${LOKI_CPUS}
|
||||
mem_limit: ${LOKI_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [edge, monitoring]
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.10.10
|
||||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
ports:
|
||||
- "${TEMPO_BIND_ADDRESS}:3200"
|
||||
healthcheck:
|
||||
test: ["CMD", "/tempo", "-version"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
cpus: ${TEMPO_CPUS}
|
||||
mem_limit: ${TEMPO_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [edge, monitoring]
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.160.0
|
||||
command: ["--config=/etc/otelcol-contrib/collector.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro
|
||||
expose:
|
||||
- "4317"
|
||||
- "4318"
|
||||
- "8888"
|
||||
- "8889"
|
||||
- "13133"
|
||||
depends_on:
|
||||
- loki
|
||||
- tempo
|
||||
healthcheck:
|
||||
test: ["CMD", "/otelcol-contrib", "components"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
cpus: ${OTEL_COLLECTOR_CPUS}
|
||||
mem_limit: ${OTEL_COLLECTOR_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [application, data, monitoring]
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
valkey_data:
|
||||
redis_broker_data:
|
||||
prometheus_data:
|
||||
grafana_data:
|
||||
loki_data:
|
||||
tempo_data:
|
||||
s3_data:
|
||||
|
||||
networks:
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_address: 0.0.0.0
|
||||
http_listen_port: 3100
|
||||
grpc_listen_address: 0.0.0.0
|
||||
grpc_listen_port: 9096
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 168h
|
||||
volume_enabled: true
|
||||
otlp_config:
|
||||
resource_attributes:
|
||||
attributes_config:
|
||||
- action: index_label
|
||||
attributes:
|
||||
- service.name
|
||||
- deployment.environment.name
|
||||
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,73 @@
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 192
|
||||
spike_limit_mib: 48
|
||||
batch:
|
||||
send_batch_size: 1024
|
||||
send_batch_max_size: 2048
|
||||
timeout: 2s
|
||||
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
resource_constant_labels:
|
||||
included:
|
||||
- service.name
|
||||
- service.version
|
||||
- deployment.environment.name
|
||||
otlp_grpc/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
insecure: true
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 512
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 30s
|
||||
otlp_http/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 512
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 30s
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
telemetry:
|
||||
logs:
|
||||
level: info
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp_http/loki]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp_grpc/tempo]
|
||||
@@ -0,0 +1,48 @@
|
||||
server:
|
||||
http_listen_address: 0.0.0.0
|
||||
http_listen_port: 3200
|
||||
grpc_listen_address: 0.0.0.0
|
||||
grpc_listen_port: 9095
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 168h
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/traces
|
||||
|
||||
metrics_generator:
|
||||
registry:
|
||||
external_labels:
|
||||
source: tempo
|
||||
deployment_environment: lab
|
||||
storage:
|
||||
path: /var/tempo/generator/wal
|
||||
remote_write:
|
||||
- url: http://prometheus:9090/api/v1/write
|
||||
send_exemplars: true
|
||||
|
||||
overrides:
|
||||
defaults:
|
||||
metrics_generator:
|
||||
processors: [service-graphs, span-metrics]
|
||||
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -1,3 +1,6 @@
|
||||
# Изолированное имя Compose-проекта и его volumes/networks.
|
||||
COMPOSE_PROJECT_NAME=mtusi2026lab6
|
||||
|
||||
# Адрес и внутренний порт HTTP-сервера.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
@@ -8,6 +11,14 @@ API_BIND_ADDRESS=127.0.0.1:8080
|
||||
SWAGGER_BIND_ADDRESS=127.0.0.1:8081
|
||||
PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090
|
||||
GRAFANA_BIND_ADDRESS=127.0.0.1:3000
|
||||
LOKI_BIND_ADDRESS=127.0.0.1:3100
|
||||
TEMPO_BIND_ADDRESS=127.0.0.1:3200
|
||||
|
||||
# Ресурсные атрибуты и внутренний OTLP endpoint приложений.
|
||||
DEPLOYMENT_ENVIRONMENT=lab
|
||||
SERVICE_VERSION=lab6
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=grpc
|
||||
|
||||
# Число API-реплик за Nginx.
|
||||
API_REPLICAS=2
|
||||
@@ -55,8 +66,14 @@ EXPORTER_CPUS=0.25
|
||||
EXPORTER_MEMORY=128m
|
||||
PROMETHEUS_CPUS=1
|
||||
PROMETHEUS_MEMORY=512m
|
||||
GRAFANA_CPUS=0.5
|
||||
GRAFANA_MEMORY=256m
|
||||
GRAFANA_CPUS=1
|
||||
GRAFANA_MEMORY=512m
|
||||
OTEL_COLLECTOR_CPUS=0.5
|
||||
OTEL_COLLECTOR_MEMORY=256m
|
||||
LOKI_CPUS=1
|
||||
LOKI_MEMORY=512m
|
||||
TEMPO_CPUS=1
|
||||
TEMPO_MEMORY=512m
|
||||
POSTGRES_CPUS=2
|
||||
POSTGRES_MEMORY=1g
|
||||
VALKEY_CPUS=1
|
||||
+602
-8
@@ -67,7 +67,7 @@
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "API latency p95 by route",
|
||||
"title": "API latency p50 / p95 / p99 by route",
|
||||
"id": 3,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
@@ -77,9 +77,19 @@
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "{{method}} {{route}}",
|
||||
"expr": "histogram_quantile(0.50, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "p50 {{method}} {{route}}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "p95 {{method}} {{route}}",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "p99 {{method}} {{route}}",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
@@ -895,14 +905,563 @@
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "SLO and alerts",
|
||||
"id": 105,
|
||||
"collapsed": true,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 109
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Business API availability",
|
||||
"id": 31,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 110
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "1 - (sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001)",
|
||||
"legendFormat": "availability",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "API error-budget burn rate (99.5% SLO)",
|
||||
"id": 32,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 110
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001) / 0.005",
|
||||
"legendFormat": "5m burn",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[1h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[1h])) or vector(0)), 0.000001) / 0.005",
|
||||
"legendFormat": "1h burn",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[6h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[6h])) or vector(0)), 0.000001) / 0.005",
|
||||
"legendFormat": "6h burn",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Attachments ready within 30 seconds",
|
||||
"id": 33,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 118
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "1 - (clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001)",
|
||||
"legendFormat": "ready in 30s",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Attachment ready-time error-budget burn",
|
||||
"id": 34,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 118
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001) / 0.01",
|
||||
"legendFormat": "5m burn",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[1h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0)), 0.000001) / 0.01",
|
||||
"legendFormat": "1h burn",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[6h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0)), 0.000001) / 0.01",
|
||||
"legendFormat": "6h burn",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "SLO alert state",
|
||||
"id": 35,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 126
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max by (alertname, alertstate) (ALERTS{alertname=~\"MessengerAPIHighErrorBudgetBurn.*|AttachmentReadyHighErrorBudgetBurn.*|AttachmentQueueBacklogGrowing|AttachmentWorkerUnavailable\"})",
|
||||
"legendFormat": "{{alertname}} {{alertstate}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Runtime resources",
|
||||
"id": 107,
|
||||
"collapsed": true,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 134
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Process CPU by service",
|
||||
"id": 38,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 135
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (service_name) (rate(process_cpu_seconds_total[1m]))",
|
||||
"legendFormat": "{{service_name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "cores"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Go heap by service",
|
||||
"id": 39,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 135
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (service_name) (go_memstats_heap_alloc_bytes)",
|
||||
"legendFormat": "{{service_name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Database pool used and wait rate",
|
||||
"id": 40,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 143
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (service_name) (messenger_db_pool_connections{state=\"used\"})",
|
||||
"legendFormat": "{{service_name}} used",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (service_name) (rate(messenger_db_pool_waited_total[5m]))",
|
||||
"legendFormat": "{{service_name}} waits/s",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Worker saturation",
|
||||
"id": 41,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 143
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(messenger_attachment_worker_jobs_in_flight)",
|
||||
"legendFormat": "busy",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum(messenger_attachment_worker_capacity)",
|
||||
"legendFormat": "capacity",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Logs and traces",
|
||||
"id": 106,
|
||||
"collapsed": true,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 151
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "logs",
|
||||
"title": "Correlated application logs",
|
||||
"id": 36,
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 152
|
||||
},
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "{service_name=~\"messenger-.+\"} | json",
|
||||
"queryType": "range",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"dedupStrategy": "none",
|
||||
"enableLogDetails": true,
|
||||
"prettifyLogMessage": false,
|
||||
"showCommonLabels": false,
|
||||
"showLabels": false,
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "traces",
|
||||
"title": "Recent application traces (TraceQL)",
|
||||
"id": 37,
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 152
|
||||
},
|
||||
"datasource": {
|
||||
"type": "tempo",
|
||||
"uid": "tempo"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "tempo",
|
||||
"uid": "tempo"
|
||||
},
|
||||
"query": "{ resource.service.name =~ \"messenger-.*\" }",
|
||||
"queryType": "traceql",
|
||||
"refId": "A",
|
||||
"limit": 50,
|
||||
"tableType": "traces"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showSpanFilters": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Trace-derived service call rate",
|
||||
"id": 42,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 162
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (service, span_name) (rate(traces_spanmetrics_calls_total[5m]))",
|
||||
"legendFormat": "{{service}} {{span_name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Collector delivery failures and queue pressure",
|
||||
"id": 43,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 162
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans_total[5m]))",
|
||||
"legendFormat": "{{exporter}} spans failed/s",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records_total[5m]))",
|
||||
"legendFormat": "{{exporter}} logs failed/s",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "max by (exporter) (otelcol_exporter_queue_size)",
|
||||
"legendFormat": "{{exporter}} queue",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "max by (exporter) (otelcol_exporter_queue_capacity)",
|
||||
"legendFormat": "{{exporter}} capacity",
|
||||
"refId": "D"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"refresh": "5s",
|
||||
"schemaVersion": 41,
|
||||
"tags": [
|
||||
"highload",
|
||||
"lab5",
|
||||
"pipeline"
|
||||
"lab6",
|
||||
"observability",
|
||||
"slo"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
@@ -912,7 +1471,42 @@
|
||||
"to": "now"
|
||||
},
|
||||
"timezone": "browser",
|
||||
"title": "Messenger lab5 overview",
|
||||
"uid": "messenger-lab5",
|
||||
"version": 3
|
||||
"title": "Messenger lab6 observability",
|
||||
"uid": "messenger-lab6",
|
||||
"version": 1,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": false,
|
||||
"icon": "external link",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": [],
|
||||
"targetBlank": false,
|
||||
"title": "Explore logs and traces",
|
||||
"type": "link",
|
||||
"url": "/explore"
|
||||
},
|
||||
{
|
||||
"asDropdown": false,
|
||||
"icon": "external link",
|
||||
"includeVars": false,
|
||||
"keepTime": false,
|
||||
"tags": [],
|
||||
"targetBlank": true,
|
||||
"title": "API runbook",
|
||||
"type": "link",
|
||||
"url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md"
|
||||
},
|
||||
{
|
||||
"asDropdown": false,
|
||||
"icon": "external link",
|
||||
"includeVars": false,
|
||||
"keepTime": false,
|
||||
"tags": [],
|
||||
"targetBlank": true,
|
||||
"title": "Pipeline runbook",
|
||||
"type": "link",
|
||||
"url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
apiVersion: 1
|
||||
|
||||
deleteDatasources:
|
||||
- name: Prometheus
|
||||
orgId: 1
|
||||
- name: Loki
|
||||
orgId: 1
|
||||
- name: Tempo
|
||||
orgId: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
exemplarTraceIdDestinations:
|
||||
- name: trace_id
|
||||
datasourceUid: tempo
|
||||
|
||||
- name: Loki
|
||||
type: loki
|
||||
uid: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
maxLines: 1000
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherRegex: 'trace_id[=:" ]+([0-9a-f]{32})'
|
||||
datasourceUid: tempo
|
||||
url: '$${__value.raw}'
|
||||
|
||||
- name: Tempo
|
||||
type: tempo
|
||||
uid: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
search:
|
||||
hide: false
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: -5m
|
||||
spanEndTimeShift: 5m
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
customQuery: true
|
||||
query: '{service_name="$${__span.tags["service.name"]}"} | trace_id = "$${__trace.traceId}"'
|
||||
tags:
|
||||
- key: service.name
|
||||
value: service_name
|
||||
tracesToMetrics:
|
||||
datasourceUid: prometheus
|
||||
spanStartTimeShift: -5m
|
||||
spanEndTimeShift: 5m
|
||||
tags:
|
||||
- key: service.name
|
||||
value: service
|
||||
queries:
|
||||
- name: Request rate
|
||||
query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))'
|
||||
- name: Error rate
|
||||
query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags,span_status="STATUS_CODE_ERROR"}[5m]))'
|
||||
@@ -1,10 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
global:
|
||||
scrape_interval: 1s
|
||||
evaluation_interval: 1s
|
||||
scrape_interval: 2s
|
||||
evaluation_interval: 5s
|
||||
external_labels:
|
||||
deployment_environment: lab
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: messenger
|
||||
@@ -9,6 +14,13 @@ scrape_configs:
|
||||
type: A
|
||||
port: 8080
|
||||
refresh_interval: 2s
|
||||
relabel_configs:
|
||||
- target_label: service_name
|
||||
replacement: messenger-api
|
||||
metric_relabel_configs:
|
||||
- source_labels: [__name__]
|
||||
regex: "go_.*|process_.*"
|
||||
action: keep
|
||||
|
||||
- job_name: nginx
|
||||
static_configs:
|
||||
@@ -32,12 +44,42 @@ scrape_configs:
|
||||
type: A
|
||||
port: 9091
|
||||
refresh_interval: 2s
|
||||
relabel_configs:
|
||||
- target_label: service_name
|
||||
replacement: messenger-worker
|
||||
metric_relabel_configs:
|
||||
- source_labels: [__name__]
|
||||
regex: "go_.*|process_.*"
|
||||
action: keep
|
||||
|
||||
- job_name: outbox-dispatcher
|
||||
static_configs:
|
||||
- targets: ["dispatcher:9091"]
|
||||
relabel_configs:
|
||||
- target_label: service_name
|
||||
replacement: messenger-dispatcher
|
||||
metric_relabel_configs:
|
||||
- source_labels: [__name__]
|
||||
regex: "go_.*|process_.*"
|
||||
action: keep
|
||||
|
||||
- job_name: minio
|
||||
metrics_path: /minio/v2/metrics/cluster
|
||||
static_configs:
|
||||
- targets: ["s3:9000"]
|
||||
|
||||
- job_name: otel-applications
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8889"]
|
||||
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8888"]
|
||||
|
||||
- job_name: tempo
|
||||
static_configs:
|
||||
- targets: ["tempo:3200"]
|
||||
|
||||
- job_name: loki
|
||||
static_configs:
|
||||
- targets: ["loki:3100"]
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
groups:
|
||||
- name: messenger-api-slo
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: MessengerAPIBusiness5xxHigh
|
||||
expr: |
|
||||
(
|
||||
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[2m])) or vector(0))
|
||||
/
|
||||
clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[2m])) or vector(0)), 0.000001)
|
||||
) > 0.05
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
service: messenger-api
|
||||
slo: api-availability
|
||||
annotations:
|
||||
summary: Business API 5xx ratio is above 5%
|
||||
description: More than 5% of business requests returned 5xx for one minute. Health and metrics routes are excluded.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
|
||||
|
||||
- alert: MessengerAPIHighErrorBudgetBurnFast
|
||||
expr: |
|
||||
(
|
||||
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[5m])) or vector(0))
|
||||
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[5m])) or vector(0)), 0.000001)
|
||||
) > 0.072
|
||||
and
|
||||
(
|
||||
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[1h])) or vector(0))
|
||||
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[1h])) or vector(0)), 0.000001)
|
||||
) > 0.072
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
service: messenger-api
|
||||
slo: api-availability
|
||||
annotations:
|
||||
summary: API availability error budget is burning fast
|
||||
description: The 99.5% API availability SLO burns above 14.4x in both 5m and 1h windows.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
|
||||
|
||||
- alert: MessengerAPIHighErrorBudgetBurnSlow
|
||||
expr: |
|
||||
(
|
||||
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[30m])) or vector(0))
|
||||
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[30m])) or vector(0)), 0.000001)
|
||||
) > 0.03
|
||||
and
|
||||
(
|
||||
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[6h])) or vector(0))
|
||||
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[6h])) or vector(0)), 0.000001)
|
||||
) > 0.03
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
service: messenger-api
|
||||
slo: api-availability
|
||||
annotations:
|
||||
summary: API availability error budget is burning steadily
|
||||
description: The 99.5% API availability SLO burns above 6x in both 30m and 6h windows.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
|
||||
|
||||
- name: messenger-attachment-slo
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: AttachmentReadyHighErrorBudgetBurnFast
|
||||
expr: |
|
||||
(
|
||||
clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0))
|
||||
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[5m])) or vector(0)),
|
||||
0
|
||||
)
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0))
|
||||
)
|
||||
/ clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0)),
|
||||
0.000001
|
||||
) > 0.144
|
||||
and
|
||||
(
|
||||
clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0))
|
||||
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h])) or vector(0)),
|
||||
0
|
||||
)
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0))
|
||||
)
|
||||
/ clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0)),
|
||||
0.000001
|
||||
) > 0.144
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
service: messenger-worker
|
||||
slo: attachment-ready
|
||||
annotations:
|
||||
summary: Attachment ready-time error budget is burning fast
|
||||
description: The 99% within 30s ready-time SLO burns above 14.4x in both 5m and 1h windows.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
||||
|
||||
- alert: AttachmentReadyHighErrorBudgetBurnSlow
|
||||
expr: |
|
||||
(
|
||||
clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0))
|
||||
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[30m])) or vector(0)),
|
||||
0
|
||||
)
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0))
|
||||
)
|
||||
/ clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0)),
|
||||
0.000001
|
||||
) > 0.06
|
||||
and
|
||||
(
|
||||
clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0))
|
||||
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[6h])) or vector(0)),
|
||||
0
|
||||
)
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0))
|
||||
)
|
||||
/ clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0)),
|
||||
0.000001
|
||||
) > 0.06
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
service: messenger-worker
|
||||
slo: attachment-ready
|
||||
annotations:
|
||||
summary: Attachment ready-time error budget is burning steadily
|
||||
description: The 99% within 30s ready-time SLO burns above 6x in both 30m and 6h windows.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
||||
|
||||
- alert: AttachmentWorkerUnavailable
|
||||
expr: (sum(up{job="attachment-worker"}) or vector(0)) < 1
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
service: messenger-worker
|
||||
annotations:
|
||||
summary: No attachment worker is available
|
||||
description: Prometheus has not observed a healthy worker target for 30 seconds.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
||||
|
||||
- alert: AttachmentQueueBacklogGrowing
|
||||
expr: |
|
||||
max(messenger_attachment_jobs_oldest_age_seconds{status="queued"}) > 30
|
||||
and max(deriv(messenger_attachment_jobs_oldest_age_seconds{status="queued"}[2m])) > 0
|
||||
for: 30s
|
||||
labels:
|
||||
severity: warning
|
||||
service: messenger-worker
|
||||
annotations:
|
||||
summary: Attachment queue backlog is growing
|
||||
description: The oldest queued attachment is older than 30 seconds and its age continues to increase.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
||||
|
||||
- alert: AttachmentRedisStreamLagHigh
|
||||
expr: max(redis_stream_group_lag{job="redis-broker",stream="attachment-jobs",group="attachment-workers"}) > 20
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
service: redis-broker
|
||||
annotations:
|
||||
summary: Redis Stream consumer lag is high
|
||||
description: The attachment worker group has more than 20 messages waiting for delivery.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
||||
@@ -0,0 +1,72 @@
|
||||
# API availability SLO
|
||||
|
||||
## Цель и сигналы
|
||||
|
||||
Объект контроля — бизнес-маршруты API за Nginx. Health endpoints исключены. Хорошими считаются ответы `2xx` и ожидаемые клиентские `4xx`; плохими — `5xx`. Цель — не менее 99,5% хороших ответов за скользящие 30 дней, error budget равен 0,5%.
|
||||
|
||||
Основные сигналы:
|
||||
|
||||
- `messenger_http_requests_total{method,route,status_class}` — число запросов;
|
||||
- `messenger_http_request_duration_seconds_bucket{method,route}` — распределение server latency;
|
||||
- `up{job="messenger"}` и `up{job="nginx"}` — доступность scrape targets;
|
||||
- `messenger_db_pool_connections{state="used"}` и `rate(messenger_db_pool_waited_total[5m])` — использование пула и частота ожиданий свободного соединения;
|
||||
- trace `messenger-api` — путь одного запроса по зависимостям;
|
||||
- JSON-логи `messenger-api` с `request_id`, `trace_id` и `span_id`.
|
||||
|
||||
`user_id`, `chat_id`, `message_id`, `attachment_id` и `request_id` не используются как labels метрик.
|
||||
|
||||
## Диагностика: metrics → trace → logs
|
||||
|
||||
1. В Grafana открыть dashboard **Messenger Lab 6 / API** и установить общий временной диапазон инцидента.
|
||||
2. Проверить error ratio, RPS, p95/p99 и `up`. Рост `5xx` при нормальном RPS подтверждает влияние на SLI; отсутствие трафика не считается успешностью.
|
||||
3. Сравнить DB pool used/waiting и состояние PostgreSQL. Рост ожидания пула указывает на зависимость, а не на CPU handler.
|
||||
4. В панели ошибок выбрать exemplar или перейти в Explore → Tempo. Если exemplar отсутствует, выполнить TraceQL:
|
||||
|
||||
```text
|
||||
{ resource.service.name = "messenger-api" && span.http.route != "/health/ready" && status = error }
|
||||
```
|
||||
|
||||
5. В trace найти первый error span. Проверить `http.route`, `http.request.method`, `http.response.status_code`, `db.system`, duration и resource attributes. Полный URL, токен и тело запроса в span отсутствуют.
|
||||
6. Скопировать `trace_id`, перейти по trace-to-logs или выполнить в Explore → Loki:
|
||||
|
||||
```text
|
||||
{service_name="messenger-api"} | json | trace_id="<trace_id>"
|
||||
```
|
||||
|
||||
7. Сверить `request_id`, событие, severity и error class. Текст сообщений, `Authorization` и пароли в логах отсутствуют.
|
||||
|
||||
## Проверка alert
|
||||
|
||||
Состояние правил видно в Grafana Alerting и в Prometheus:
|
||||
|
||||
```bash
|
||||
curl -s http://127.0.0.1:9090/api/v1/alerts
|
||||
```
|
||||
|
||||
Контролируемый отказ PostgreSQL:
|
||||
|
||||
```bash
|
||||
ENV_FILE=lab6.env.example scripts/incident-database.sh
|
||||
```
|
||||
|
||||
Сценарий сохраняет baseline, нагрузку, состояние деградации, `alert-firing.json`, восстановление и `alert-resolved.json` в `evidence/runtime/`. Trap запускает PostgreSQL и API даже при ошибке сценария. Volumes не удаляются.
|
||||
|
||||
Ожидаемая последовательность:
|
||||
|
||||
```text
|
||||
нормальный error ratio
|
||||
→ PostgreSQL остановлен
|
||||
→ 5xx растёт
|
||||
→ MessengerAPIBusiness5xxHigh и MessengerAPIHighErrorBudgetBurnFast firing
|
||||
→ PostgreSQL восстановлен
|
||||
→ успешные ответы возвращаются
|
||||
→ оперативный alert resolved; burn-rate хранит историю длинного окна
|
||||
```
|
||||
|
||||
## Действия
|
||||
|
||||
1. Если `up{job="nginx"}=0`, восстановить ingress и проверить его логи.
|
||||
2. Если API targets недоступны, восстановить реплики по одной и проверить readiness.
|
||||
3. Если API работает, а DB spans завершаются ошибкой или pool waiting растёт, восстановить PostgreSQL и не увеличивать число API-реплик до устранения причины.
|
||||
4. После восстановления проверить успешный smoke, отсутствие firing alerts и возвращение latency к baseline.
|
||||
5. Не удалять volumes и не сбрасывать очередь ради закрытия alert.
|
||||
@@ -0,0 +1,85 @@
|
||||
# Attachment pipeline SLO
|
||||
|
||||
## Цель и сигналы
|
||||
|
||||
Объект контроля — время от durable acceptance изображения до состояния `ready`. Цель — не менее 99% jobs завершаются `ready` не позднее 30 секунд за скользящие 30 дней. `failed` и завершение позже 30 секунд расходуют error budget.
|
||||
|
||||
Основные сигналы:
|
||||
|
||||
- `messenger_attachment_worker_jobs_total{job_type,result}` — исходы worker;
|
||||
- `messenger_attachment_worker_job_duration_seconds_bucket{job_type,result}` — время обработки;
|
||||
- `messenger_attachment_time_to_ready_seconds_bucket` — end-to-end время;
|
||||
- `messenger_attachment_jobs_count{status}` — состояния jobs в PostgreSQL;
|
||||
- `messenger_attachment_jobs_oldest_age_seconds{status}` — возраст старейшей job;
|
||||
- `messenger_attachment_worker_jobs_in_flight` и `messenger_attachment_worker_capacity` — насыщение worker;
|
||||
- `messenger_outbox_pending` — задержка до публикации;
|
||||
- Redis Stream lag и PEL — доставка и неподтверждённые сообщения.
|
||||
|
||||
Queue depth не заменяет processing latency: очередь может быть короткой, но содержать одну очень старую job.
|
||||
|
||||
## Диагностика: metrics → trace → logs
|
||||
|
||||
1. В Grafana открыть dashboard **Messenger Lab 6 / Attachment pipeline** и выбрать диапазон инцидента.
|
||||
2. Определить место задержки:
|
||||
- растёт `messenger_outbox_pending` — проблема до публикации в broker;
|
||||
- растут stream lag и `queued` — worker не успевает или недоступен;
|
||||
- растут PEL и `processing` — delivery получена, но не завершена;
|
||||
- растёт `failed` — постоянные ошибки или исчерпаны retry;
|
||||
- busy равен capacity — worker насыщен.
|
||||
3. В Explore → Tempo найти upload по route или worker consumer span:
|
||||
|
||||
```text
|
||||
{ resource.service.name = "messenger-api" && span.http.route = "/api/v1/chats/{chat_id}/attachments" }
|
||||
```
|
||||
|
||||
```text
|
||||
{ resource.service.name = "messenger-worker" && name = "attachment.process" }
|
||||
```
|
||||
|
||||
4. В trace пройти цепочку `HTTP upload → outbox publish → Redis consume → S3 variants → PostgreSQL ready`. Проверить producer/consumer связь и retry spans. Технический `attachment_id` допустим только как span/log attribute синтетического стенда, но не как metric label.
|
||||
5. Перейти из выбранного span в Loki или выполнить:
|
||||
|
||||
```text
|
||||
{service_name="messenger-worker"} | json | trace_id="<trace_id>"
|
||||
```
|
||||
|
||||
6. Сверить в логе `event`, `job_type`, `attempt`, `result`, `trace_id` и `span_id`; в span проверить `job.outcome`. Payload envelope, изображение и credentials не логируются.
|
||||
|
||||
## Проверка alert
|
||||
|
||||
Короткая деградация:
|
||||
|
||||
```bash
|
||||
ENV_FILE=lab6.env.example scripts/incident-worker.sh short
|
||||
```
|
||||
|
||||
Длительная деградация:
|
||||
|
||||
```bash
|
||||
ENV_FILE=lab6.env.example scripts/incident-worker.sh long
|
||||
```
|
||||
|
||||
Ожидаемая последовательность:
|
||||
|
||||
```text
|
||||
worker up, backlog около нуля
|
||||
→ worker остановлен
|
||||
→ queued и oldest age растут
|
||||
→ short: AttachmentWorkerUnavailable firing
|
||||
→ long: AttachmentQueueBacklogGrowing firing
|
||||
→ worker запущен
|
||||
→ backlog дренируется
|
||||
→ все принятые attachments достигают ready
|
||||
→ alert resolved
|
||||
```
|
||||
|
||||
Каждый сценарий восстанавливает worker через trap и сохраняет evidence без session token и raw payload.
|
||||
|
||||
## Действия
|
||||
|
||||
1. Если worker `up=0`, восстановить worker и наблюдать скорость дренирования.
|
||||
2. Если worker доступен, но busy равно capacity, проверить S3/DB spans до масштабирования concurrency.
|
||||
3. Если растёт outbox, проверить dispatcher и PostgreSQL lease; Redis worker здесь ещё не причина.
|
||||
4. Если растёт PEL, проверить heartbeat, lease expiry и reclaim.
|
||||
5. Для `failed` определить permanent/transient error по логу и trace. Replay выполнять только после устранения причины.
|
||||
6. После восстановления проверить backlog, oldest age, `failed`, firing alerts и terminal state каждого синтетического attachment.
|
||||
Reference in New Issue
Block a user