Files
mtusi2026lab6/monitoring/rules/messenger.yml
T

179 lines
8.5 KiB
YAML

groups:
- name: messenger-api-slo
interval: 5s
rules:
- alert: MessengerAPIBusiness5xxHigh
expr: |
(
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[2m])) or vector(0))
/
clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[2m])) or vector(0)), 0.000001)
) > 0.05
for: 1m
labels:
severity: warning
service: messenger-api
slo: api-availability
annotations:
summary: Business API 5xx ratio is above 5%
description: More than 5% of business requests returned 5xx for one minute. Health and metrics routes are excluded.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
- alert: MessengerAPIHighErrorBudgetBurnFast
expr: |
(
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[5m])) or vector(0))
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[5m])) or vector(0)), 0.000001)
) > 0.072
and
(
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[1h])) or vector(0))
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[1h])) or vector(0)), 0.000001)
) > 0.072
for: 30s
labels:
severity: critical
service: messenger-api
slo: api-availability
annotations:
summary: API availability error budget is burning fast
description: The 99.5% API availability SLO burns above 14.4x in both 5m and 1h windows.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
- alert: MessengerAPIHighErrorBudgetBurnSlow
expr: |
(
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[30m])) or vector(0))
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[30m])) or vector(0)), 0.000001)
) > 0.03
and
(
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[6h])) or vector(0))
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[6h])) or vector(0)), 0.000001)
) > 0.03
for: 1m
labels:
severity: warning
service: messenger-api
slo: api-availability
annotations:
summary: API availability error budget is burning steadily
description: The 99.5% API availability SLO burns above 6x in both 30m and 6h windows.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
- name: messenger-attachment-slo
interval: 5s
rules:
- alert: AttachmentReadyHighErrorBudgetBurnFast
expr: |
(
clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0))
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[5m])) or vector(0)),
0
)
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0))
)
/ clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0))
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0)),
0.000001
) > 0.144
and
(
clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0))
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h])) or vector(0)),
0
)
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0))
)
/ clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0))
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0)),
0.000001
) > 0.144
for: 30s
labels:
severity: critical
service: messenger-worker
slo: attachment-ready
annotations:
summary: Attachment ready-time error budget is burning fast
description: The 99% within 30s ready-time SLO burns above 14.4x in both 5m and 1h windows.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
- alert: AttachmentReadyHighErrorBudgetBurnSlow
expr: |
(
clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0))
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[30m])) or vector(0)),
0
)
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0))
)
/ clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0))
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0)),
0.000001
) > 0.06
and
(
clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0))
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[6h])) or vector(0)),
0
)
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0))
)
/ clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0))
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0)),
0.000001
) > 0.06
for: 1m
labels:
severity: warning
service: messenger-worker
slo: attachment-ready
annotations:
summary: Attachment ready-time error budget is burning steadily
description: The 99% within 30s ready-time SLO burns above 6x in both 30m and 6h windows.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
- alert: AttachmentWorkerUnavailable
expr: (sum(up{job="attachment-worker"}) or vector(0)) < 1
for: 30s
labels:
severity: critical
service: messenger-worker
annotations:
summary: No attachment worker is available
description: Prometheus has not observed a healthy worker target for 30 seconds.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
- alert: AttachmentQueueBacklogGrowing
expr: |
max(messenger_attachment_jobs_oldest_age_seconds{status="queued"}) > 30
and max(deriv(messenger_attachment_jobs_oldest_age_seconds{status="queued"}[2m])) > 0
for: 30s
labels:
severity: warning
service: messenger-worker
annotations:
summary: Attachment queue backlog is growing
description: The oldest queued attachment is older than 30 seconds and its age continues to increase.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
- alert: AttachmentRedisStreamLagHigh
expr: max(redis_stream_group_lag{job="redis-broker",stream="attachment-jobs",group="attachment-workers"}) > 20
for: 1m
labels:
severity: warning
service: redis-broker
annotations:
summary: Redis Stream consumer lag is high
description: The attachment worker group has more than 20 messages waiting for delivery.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md