28 lines
1.1 KiB
YAML
28 lines
1.1 KiB
YAML
rule_files:
|
|
- /workspace/infra/prometheus/alerts.yaml
|
|
evaluation_interval: 5s
|
|
tests:
|
|
- interval: 5s
|
|
input_series:
|
|
- series: 'messenger_http_requests_total{method="POST",route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}'
|
|
values: '0+50x30'
|
|
- series: 'messenger_http_requests_total{method="POST",route="/api/v1/chats/{chat_id}/messages",status_class="2xx"}'
|
|
values: '0+450x30'
|
|
- series: 'messenger_queue_age_seconds{service_name="messenger-outbox"}'
|
|
values: '60x30'
|
|
alert_rule_test:
|
|
- eval_time: 2m
|
|
alertname: MessengerHighErrorRate
|
|
exp_alerts:
|
|
- exp_labels: {severity: warning}
|
|
exp_annotations:
|
|
summary: 'API 5xx ratio exceeds 5%'
|
|
runbook: 'Inspect API latency, traces and dependency spans; check DB readiness.'
|
|
- eval_time: 2m
|
|
alertname: MessengerQueueLag
|
|
exp_alerts:
|
|
- exp_labels: {severity: warning}
|
|
exp_annotations:
|
|
summary: 'Oldest queued job exceeds 30 seconds'
|
|
runbook: 'Check worker health, busy/capacity, broker and consumer traces.'
|