forked from mtusi2026/mtusi2026lab7
Закрывает обязательный минимум и трек 1: ротация leaf и секрета, sealed OpenBao и сохранность queued-задачи.
97 lines
5.0 KiB
YAML
97 lines
5.0 KiB
YAML
groups:
|
|
- name: messenger-sli
|
|
interval: 5s
|
|
rules:
|
|
- record: messenger:api_requests:rate1m
|
|
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[1m]))
|
|
- record: messenger:api_5xx:rate1m
|
|
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[1m]))
|
|
- record: messenger:api_error_ratio:rate1m
|
|
expr: messenger:api_5xx:rate1m / clamp_min(messenger:api_requests:rate1m, 1e-9)
|
|
- record: messenger:api_requests:rate5m
|
|
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[5m]))
|
|
- record: messenger:api_5xx:rate5m
|
|
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[5m]))
|
|
- record: messenger:api_error_ratio:rate5m
|
|
expr: messenger:api_5xx:rate5m / clamp_min(messenger:api_requests:rate5m, 1e-9)
|
|
- record: messenger:api_requests:rate15m
|
|
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[15m]))
|
|
- record: messenger:api_5xx:rate15m
|
|
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[15m]))
|
|
- record: messenger:api_error_ratio:rate15m
|
|
expr: messenger:api_5xx:rate15m / clamp_min(messenger:api_requests:rate15m, 1e-9)
|
|
|
|
- name: messenger-alerts
|
|
interval: 5s
|
|
rules:
|
|
- alert: ApiHighErrorRate
|
|
expr: messenger:api_error_ratio:rate5m > 0.05 and messenger:api_requests:rate5m > 0.2
|
|
for: 1m
|
|
labels:
|
|
severity: page
|
|
slo: api-availability
|
|
annotations:
|
|
summary: API business 5xx ratio is above 5%
|
|
description: Numerator is 5xx on business routes. Denominator is all business requests. Health checks and 4xx are excluded. Window 5m, for 1m.
|
|
runbook: docs/runbooks/api-error-budget.md
|
|
- alert: ApiErrorBudgetFastBurn
|
|
expr: messenger:api_error_ratio:rate1m > 0.144 and messenger:api_error_ratio:rate5m > 0.144 and messenger:api_requests:rate1m > 0.2
|
|
for: 30s
|
|
labels:
|
|
severity: page
|
|
slo: api-availability
|
|
burn: fast
|
|
annotations:
|
|
summary: Fast multi-window burn of the 1% API error budget
|
|
description: Both the 1m and 5m error ratios exceed 14.4% (14.4x a 1% budget). A sub-minute spike does not satisfy the 5m window when healthy traffic is present.
|
|
runbook: docs/runbooks/api-error-budget.md
|
|
- alert: ApiErrorBudgetSlowBurn
|
|
expr: messenger:api_error_ratio:rate5m > 0.06 and messenger:api_error_ratio:rate15m > 0.06 and messenger:api_requests:rate5m > 0.2
|
|
for: 1m
|
|
labels:
|
|
severity: ticket
|
|
slo: api-availability
|
|
burn: slow
|
|
annotations:
|
|
summary: Slow multi-window burn of the API error budget
|
|
description: 5m and 15m error ratios exceed 6% (6x a 1% budget). This catches a long mild degradation that the fast burn may miss.
|
|
runbook: docs/runbooks/api-error-budget.md
|
|
- alert: QueueLagHigh
|
|
expr: max(messenger_queue_oldest_age_seconds{queue="attachment.process.v1"}) > 30
|
|
for: 1m
|
|
labels:
|
|
severity: page
|
|
slo: time-to-ready
|
|
annotations:
|
|
summary: Oldest attachment job is older than 30s
|
|
description: Backlog age is not processing latency. It grows when the worker is stopped or cannot finish jobs. Unit is seconds.
|
|
runbook: docs/runbooks/queue-lag.md
|
|
- alert: WorkerConsumersZero
|
|
expr: max(messenger_worker_consumers{service_name="messenger-api",queue="attachment.process.v1"}) == 0
|
|
for: 1m
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
summary: RabbitMQ has no attachment consumers
|
|
description: The API polls the RabbitMQ management API, so this series stays fresh after the worker process stops.
|
|
runbook: docs/runbooks/queue-lag.md
|
|
- alert: AttachmentReadyLatencyHigh
|
|
expr: histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[15m]))) > 10
|
|
for: 2m
|
|
labels:
|
|
severity: ticket
|
|
slo: time-to-ready
|
|
annotations:
|
|
summary: p95 time to ready is above 10s
|
|
description: Histogram of completed jobs only. A stalled queue does not move this SLO; use QueueLagHigh for that.
|
|
runbook: docs/runbooks/time-to-ready.md
|
|
- alert: DatabaseUnready
|
|
expr: min(messenger_db_up) == 0
|
|
for: 30s
|
|
labels:
|
|
severity: page
|
|
annotations:
|
|
summary: An API or worker process cannot use PostgreSQL
|
|
description: Distinguish this from a slow handler by comparing the db.transaction span with the server span.
|
|
runbook: docs/runbooks/db-latency.md
|