Files
mtusi2026lab7/infra/prometheus/rules.yml
T
vika baafe2a3ae Добавить стенд лабораторной 7 с mTLS, OpenBao и least privilege worker.
Закрывает обязательный минимум и трек 1: ротация leaf и секрета, sealed OpenBao и сохранность queued-задачи.
2026-11-07 20:46:02 +03:00

97 lines
5.0 KiB
YAML

groups:
- name: messenger-sli
interval: 5s
rules:
- record: messenger:api_requests:rate1m
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[1m]))
- record: messenger:api_5xx:rate1m
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[1m]))
- record: messenger:api_error_ratio:rate1m
expr: messenger:api_5xx:rate1m / clamp_min(messenger:api_requests:rate1m, 1e-9)
- record: messenger:api_requests:rate5m
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[5m]))
- record: messenger:api_5xx:rate5m
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[5m]))
- record: messenger:api_error_ratio:rate5m
expr: messenger:api_5xx:rate5m / clamp_min(messenger:api_requests:rate5m, 1e-9)
- record: messenger:api_requests:rate15m
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[15m]))
- record: messenger:api_5xx:rate15m
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[15m]))
- record: messenger:api_error_ratio:rate15m
expr: messenger:api_5xx:rate15m / clamp_min(messenger:api_requests:rate15m, 1e-9)
- name: messenger-alerts
interval: 5s
rules:
- alert: ApiHighErrorRate
expr: messenger:api_error_ratio:rate5m > 0.05 and messenger:api_requests:rate5m > 0.2
for: 1m
labels:
severity: page
slo: api-availability
annotations:
summary: API business 5xx ratio is above 5%
description: Numerator is 5xx on business routes. Denominator is all business requests. Health checks and 4xx are excluded. Window 5m, for 1m.
runbook: docs/runbooks/api-error-budget.md
- alert: ApiErrorBudgetFastBurn
expr: messenger:api_error_ratio:rate1m > 0.144 and messenger:api_error_ratio:rate5m > 0.144 and messenger:api_requests:rate1m > 0.2
for: 30s
labels:
severity: page
slo: api-availability
burn: fast
annotations:
summary: Fast multi-window burn of the 1% API error budget
description: Both the 1m and 5m error ratios exceed 14.4% (14.4x a 1% budget). A sub-minute spike does not satisfy the 5m window when healthy traffic is present.
runbook: docs/runbooks/api-error-budget.md
- alert: ApiErrorBudgetSlowBurn
expr: messenger:api_error_ratio:rate5m > 0.06 and messenger:api_error_ratio:rate15m > 0.06 and messenger:api_requests:rate5m > 0.2
for: 1m
labels:
severity: ticket
slo: api-availability
burn: slow
annotations:
summary: Slow multi-window burn of the API error budget
description: 5m and 15m error ratios exceed 6% (6x a 1% budget). This catches a long mild degradation that the fast burn may miss.
runbook: docs/runbooks/api-error-budget.md
- alert: QueueLagHigh
expr: max(messenger_queue_oldest_age_seconds{queue="attachment.process.v1"}) > 30
for: 1m
labels:
severity: page
slo: time-to-ready
annotations:
summary: Oldest attachment job is older than 30s
description: Backlog age is not processing latency. It grows when the worker is stopped or cannot finish jobs. Unit is seconds.
runbook: docs/runbooks/queue-lag.md
- alert: WorkerConsumersZero
expr: max(messenger_worker_consumers{service_name="messenger-api",queue="attachment.process.v1"}) == 0
for: 1m
labels:
severity: page
annotations:
summary: RabbitMQ has no attachment consumers
description: The API polls the RabbitMQ management API, so this series stays fresh after the worker process stops.
runbook: docs/runbooks/queue-lag.md
- alert: AttachmentReadyLatencyHigh
expr: histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[15m]))) > 10
for: 2m
labels:
severity: ticket
slo: time-to-ready
annotations:
summary: p95 time to ready is above 10s
description: Histogram of completed jobs only. A stalled queue does not move this SLO; use QueueLagHigh for that.
runbook: docs/runbooks/time-to-ready.md
- alert: DatabaseUnready
expr: min(messenger_db_up) == 0
for: 30s
labels:
severity: page
annotations:
summary: An API or worker process cannot use PostgreSQL
description: Distinguish this from a slow handler by comparing the db.transaction span with the server span.
runbook: docs/runbooks/db-latency.md