forked from mtusi2026/mtusi2026lab6
feat(prometheus): lab6 alert on API errors and growing attachment queue
This commit is contained in:
@@ -342,6 +342,7 @@ services:
|
||||
- --storage.tsdb.retention.time=7d
|
||||
volumes:
|
||||
- ./infra/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
|
||||
- prometheus_data:/prometheus
|
||||
networks:
|
||||
- internal
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/messenger.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
groups:
|
||||
- name: messenger
|
||||
interval: 15s
|
||||
rules:
|
||||
- alert: MessengerHighAPIErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(http_requests_total{service_name="messenger-api",http_route=~"/api/.*",http_response_status_class="5xx"}[5m]))
|
||||
/
|
||||
sum(rate(http_requests_total{service_name="messenger-api",http_route=~"/api/.*"}[5m]))
|
||||
) > 0.05
|
||||
and
|
||||
sum(rate(http_requests_total{service_name="messenger-api",http_route=~"/api/.*"}[5m])) >= 0.2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Повышенная доля ответов 5xx API"
|
||||
description: "Доля 5xx выше 5% при нагрузке от 0,2 запроса/с за последние 5 минут."
|
||||
runbook: "REPORT.md#высокая-доля-ошибок-api"
|
||||
|
||||
- alert: MessengerGrowingAttachmentQueue
|
||||
expr: |
|
||||
rabbitmq_detailed_queue_messages_ready{job="rabbitmq",vhost="/",queue="attachments.process"} >= 5
|
||||
and
|
||||
deriv(rabbitmq_detailed_queue_messages_ready{job="rabbitmq",vhost="/",queue="attachments.process"}[5m]) > 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Растёт очередь обработки вложений"
|
||||
description: "В attachments.process ожидают как минимум 5 заданий и глубина очереди растёт в пятиминутном окне."
|
||||
runbook: "REPORT.md#растущая-очередь-вложений"
|
||||
Reference in New Issue
Block a user