feat(prometheus): lab6 alert on API errors and growing attachment queue

This commit is contained in:
Голиков Maximillian Антон
2026-11-18 09:44:27 +03:00
parent ada88ee980
commit 509b5eabf7
3 changed files with 37 additions and 0 deletions
+1
View File
@@ -342,6 +342,7 @@ services:
- --storage.tsdb.retention.time=7d
volumes:
- ./infra/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
- prometheus_data:/prometheus
networks:
- internal
+3
View File
@@ -1,6 +1,9 @@
global:
scrape_interval: 15s
rule_files:
- /etc/prometheus/rules/messenger.yml
scrape_configs:
- job_name: otel-collector
static_configs:
+33
View File
@@ -0,0 +1,33 @@
groups:
- name: messenger
interval: 15s
rules:
- alert: MessengerHighAPIErrorRate
expr: |
(
sum(rate(http_requests_total{service_name="messenger-api",http_route=~"/api/.*",http_response_status_class="5xx"}[5m]))
/
sum(rate(http_requests_total{service_name="messenger-api",http_route=~"/api/.*"}[5m]))
) > 0.05
and
sum(rate(http_requests_total{service_name="messenger-api",http_route=~"/api/.*"}[5m])) >= 0.2
for: 2m
labels:
severity: warning
annotations:
summary: "Повышенная доля ответов 5xx API"
description: "Доля 5xx выше 5% при нагрузке от 0,2 запроса/с за последние 5 минут."
runbook: "REPORT.md#высокая-доля-ошибок-api"
- alert: MessengerGrowingAttachmentQueue
expr: |
rabbitmq_detailed_queue_messages_ready{job="rabbitmq",vhost="/",queue="attachments.process"} >= 5
and
deriv(rabbitmq_detailed_queue_messages_ready{job="rabbitmq",vhost="/",queue="attachments.process"}[5m]) > 0
for: 2m
labels:
severity: warning
annotations:
summary: "Растёт очередь обработки вложений"
description: "В attachments.process ожидают как минимум 5 заданий и глубина очереди растёт в пятиминутном окне."
runbook: "REPORT.md#растущая-очередь-вложений"