From 509b5eabf739d5a8a219caa24e0fa36108d2acfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=93=D0=BE=D0=BB=D0=B8=D0=BA=D0=BE=D0=B2=20Maximillian?= =?UTF-8?q?=20=D0=90=D0=BD=D1=82=D0=BE=D0=BD?= Date: Wed, 18 Nov 2026 09:44:27 +0300 Subject: [PATCH] feat(prometheus): lab6 alert on API errors and growing attachment queue --- compose.yaml | 1 + infra/prometheus.yml | 3 +++ infra/prometheus/rules/messenger.yml | 33 ++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 infra/prometheus/rules/messenger.yml diff --git a/compose.yaml b/compose.yaml index d58a033..6a4b63e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -342,6 +342,7 @@ services: - --storage.tsdb.retention.time=7d volumes: - ./infra/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/rules:/etc/prometheus/rules:ro - prometheus_data:/prometheus networks: - internal diff --git a/infra/prometheus.yml b/infra/prometheus.yml index 2cd1061..da135e6 100644 --- a/infra/prometheus.yml +++ b/infra/prometheus.yml @@ -1,6 +1,9 @@ global: scrape_interval: 15s +rule_files: + - /etc/prometheus/rules/messenger.yml + scrape_configs: - job_name: otel-collector static_configs: diff --git a/infra/prometheus/rules/messenger.yml b/infra/prometheus/rules/messenger.yml new file mode 100644 index 0000000..fb1fe01 --- /dev/null +++ b/infra/prometheus/rules/messenger.yml @@ -0,0 +1,33 @@ +groups: + - name: messenger + interval: 15s + rules: + - alert: MessengerHighAPIErrorRate + expr: | + ( + sum(rate(http_requests_total{service_name="messenger-api",http_route=~"/api/.*",http_response_status_class="5xx"}[5m])) + / + sum(rate(http_requests_total{service_name="messenger-api",http_route=~"/api/.*"}[5m])) + ) > 0.05 + and + sum(rate(http_requests_total{service_name="messenger-api",http_route=~"/api/.*"}[5m])) >= 0.2 + for: 2m + labels: + severity: warning + annotations: + summary: "Повышенная доля ответов 5xx API" + description: "Доля 5xx выше 5% при нагрузке от 0,2 запроса/с за последние 5 минут." + runbook: "REPORT.md#высокая-доля-ошибок-api" + + - alert: MessengerGrowingAttachmentQueue + expr: | + rabbitmq_detailed_queue_messages_ready{job="rabbitmq",vhost="/",queue="attachments.process"} >= 5 + and + deriv(rabbitmq_detailed_queue_messages_ready{job="rabbitmq",vhost="/",queue="attachments.process"}[5m]) > 0 + for: 2m + labels: + severity: warning + annotations: + summary: "Растёт очередь обработки вложений" + description: "В attachments.process ожидают как минимум 5 заданий и глубина очереди растёт в пятиминутном окне." + runbook: "REPORT.md#растущая-очередь-вложений"