84 lines
3.8 KiB
YAML
84 lines
3.8 KiB
YAML
rule_files:
|
|
- /workspace/infra/prometheus/alerts.yaml
|
|
evaluation_interval: 5s
|
|
tests:
|
|
- name: healthy
|
|
interval: 5s
|
|
input_series:
|
|
- series: 'messenger_http_requests_total{route="/api/v1/users",status_class="2xx"}'
|
|
values: '0+100x80'
|
|
- series: 'messenger_http_requests_total{route="/api/v1/users",status_class="5xx"}'
|
|
values: '0+0x80'
|
|
- series: 'messenger_slo_ready_eligible_total{service_name="messenger-outbox"}'
|
|
values: '0+100x80'
|
|
- series: 'messenger_slo_ready_bad_total{service_name="messenger-outbox"}'
|
|
values: '0+0x80'
|
|
alert_rule_test:
|
|
- {eval_time: 5m, alertname: MessengerApiBurnFast, exp_alerts: []}
|
|
- {eval_time: 5m, alertname: MessengerApiBurnSlow, exp_alerts: []}
|
|
- {eval_time: 5m, alertname: MessengerReadyBurnFast, exp_alerts: []}
|
|
- {eval_time: 5m, alertname: MessengerReadyBurnSlow, exp_alerts: []}
|
|
- name: brief severe spike and recovery
|
|
interval: 5s
|
|
input_series:
|
|
- series: 'messenger_http_requests_total{route="/api/v1/users",status_class="2xx"}'
|
|
values: '0+100x12 1200+0x8 1200+100x60'
|
|
- series: 'messenger_http_requests_total{route="/api/v1/users",status_class="5xx"}'
|
|
values: '0+0x12 0+100x8 800+0x60'
|
|
alert_rule_test:
|
|
- eval_time: 1m30s
|
|
alertname: MessengerApiBurnFast
|
|
exp_alerts:
|
|
- exp_labels: {severity: critical, slo: api, burn: fast}
|
|
exp_annotations: {summary: 'api SLO fast burn', runbook: 'See docs/slo-runbook.md'}
|
|
- {eval_time: 3m, alertname: MessengerApiBurnFast, exp_alerts: []}
|
|
- name: sustained moderate degradation
|
|
interval: 5s
|
|
input_series:
|
|
- series: 'messenger_http_requests_total{route="/api/v1/users",status_class="2xx"}'
|
|
values: '0+80x80'
|
|
- series: 'messenger_http_requests_total{route="/api/v1/users",status_class="5xx"}'
|
|
values: '0+20x80'
|
|
- series: 'messenger_slo_ready_eligible_total{service_name="messenger-outbox"}'
|
|
values: '0+100x80'
|
|
- series: 'messenger_slo_ready_bad_total{service_name="messenger-outbox"}'
|
|
values: '0+20x80'
|
|
alert_rule_test:
|
|
- {eval_time: 5m, alertname: MessengerApiBurnFast, exp_alerts: []}
|
|
- {eval_time: 5m, alertname: MessengerReadyBurnFast, exp_alerts: []}
|
|
- eval_time: 5m
|
|
alertname: MessengerApiBurnSlow
|
|
exp_alerts:
|
|
- exp_labels: {severity: warning, slo: api, burn: slow}
|
|
exp_annotations: {summary: 'api SLO slow burn', runbook: 'See docs/slo-runbook.md'}
|
|
- eval_time: 5m
|
|
alertname: MessengerReadyBurnSlow
|
|
exp_alerts:
|
|
- exp_labels: {severity: warning, slo: ready, burn: slow}
|
|
exp_annotations: {summary: 'ready SLO slow burn', runbook: 'See docs/slo-runbook.md'}
|
|
- name: low-volume guard
|
|
interval: 5s
|
|
input_series:
|
|
- series: 'messenger_http_requests_total{route="/api/v1/users",status_class="5xx"}'
|
|
values: '0+1x80'
|
|
- series: 'messenger_slo_ready_eligible_total{service_name="messenger-outbox"}'
|
|
values: '0+1x80'
|
|
- series: 'messenger_slo_ready_bad_total{service_name="messenger-outbox"}'
|
|
values: '0+1x80'
|
|
alert_rule_test:
|
|
- {eval_time: 5m, alertname: MessengerApiBurnFast, exp_alerts: []}
|
|
- {eval_time: 5m, alertname: MessengerReadyBurnFast, exp_alerts: []}
|
|
- name: late ready events fast burn
|
|
interval: 5s
|
|
input_series:
|
|
- series: 'messenger_slo_ready_eligible_total{service_name="messenger-outbox"}'
|
|
values: '0+100x30'
|
|
- series: 'messenger_slo_ready_bad_total{service_name="messenger-outbox"}'
|
|
values: '0+100x30'
|
|
alert_rule_test:
|
|
- eval_time: 1m
|
|
alertname: MessengerReadyBurnFast
|
|
exp_alerts:
|
|
- exp_labels: {severity: critical, slo: ready, burn: fast}
|
|
exp_annotations: {summary: 'ready SLO fast burn', runbook: 'See docs/slo-runbook.md'}
|