179 lines
8.5 KiB
YAML
179 lines
8.5 KiB
YAML
groups:
|
|
- name: messenger-api-slo
|
|
interval: 5s
|
|
rules:
|
|
- alert: MessengerAPIBusiness5xxHigh
|
|
expr: |
|
|
(
|
|
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[2m])) or vector(0))
|
|
/
|
|
clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[2m])) or vector(0)), 0.000001)
|
|
) > 0.05
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
service: messenger-api
|
|
slo: api-availability
|
|
annotations:
|
|
summary: Business API 5xx ratio is above 5%
|
|
description: More than 5% of business requests returned 5xx for one minute. Health and metrics routes are excluded.
|
|
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
|
|
|
|
- alert: MessengerAPIHighErrorBudgetBurnFast
|
|
expr: |
|
|
(
|
|
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[5m])) or vector(0))
|
|
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[5m])) or vector(0)), 0.000001)
|
|
) > 0.072
|
|
and
|
|
(
|
|
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[1h])) or vector(0))
|
|
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[1h])) or vector(0)), 0.000001)
|
|
) > 0.072
|
|
for: 30s
|
|
labels:
|
|
severity: critical
|
|
service: messenger-api
|
|
slo: api-availability
|
|
annotations:
|
|
summary: API availability error budget is burning fast
|
|
description: The 99.5% API availability SLO burns above 14.4x in both 5m and 1h windows.
|
|
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
|
|
|
|
- alert: MessengerAPIHighErrorBudgetBurnSlow
|
|
expr: |
|
|
(
|
|
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[30m])) or vector(0))
|
|
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[30m])) or vector(0)), 0.000001)
|
|
) > 0.03
|
|
and
|
|
(
|
|
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[6h])) or vector(0))
|
|
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[6h])) or vector(0)), 0.000001)
|
|
) > 0.03
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
service: messenger-api
|
|
slo: api-availability
|
|
annotations:
|
|
summary: API availability error budget is burning steadily
|
|
description: The 99.5% API availability SLO burns above 6x in both 30m and 6h windows.
|
|
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
|
|
|
|
- name: messenger-attachment-slo
|
|
interval: 5s
|
|
rules:
|
|
- alert: AttachmentReadyHighErrorBudgetBurnFast
|
|
expr: |
|
|
(
|
|
clamp_min(
|
|
(sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0))
|
|
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[5m])) or vector(0)),
|
|
0
|
|
)
|
|
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0))
|
|
)
|
|
/ clamp_min(
|
|
(sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0))
|
|
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0)),
|
|
0.000001
|
|
) > 0.144
|
|
and
|
|
(
|
|
clamp_min(
|
|
(sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0))
|
|
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h])) or vector(0)),
|
|
0
|
|
)
|
|
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0))
|
|
)
|
|
/ clamp_min(
|
|
(sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0))
|
|
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0)),
|
|
0.000001
|
|
) > 0.144
|
|
for: 30s
|
|
labels:
|
|
severity: critical
|
|
service: messenger-worker
|
|
slo: attachment-ready
|
|
annotations:
|
|
summary: Attachment ready-time error budget is burning fast
|
|
description: The 99% within 30s ready-time SLO burns above 14.4x in both 5m and 1h windows.
|
|
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
|
|
|
- alert: AttachmentReadyHighErrorBudgetBurnSlow
|
|
expr: |
|
|
(
|
|
clamp_min(
|
|
(sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0))
|
|
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[30m])) or vector(0)),
|
|
0
|
|
)
|
|
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0))
|
|
)
|
|
/ clamp_min(
|
|
(sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0))
|
|
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0)),
|
|
0.000001
|
|
) > 0.06
|
|
and
|
|
(
|
|
clamp_min(
|
|
(sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0))
|
|
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[6h])) or vector(0)),
|
|
0
|
|
)
|
|
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0))
|
|
)
|
|
/ clamp_min(
|
|
(sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0))
|
|
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0)),
|
|
0.000001
|
|
) > 0.06
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
service: messenger-worker
|
|
slo: attachment-ready
|
|
annotations:
|
|
summary: Attachment ready-time error budget is burning steadily
|
|
description: The 99% within 30s ready-time SLO burns above 6x in both 30m and 6h windows.
|
|
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
|
|
|
- alert: AttachmentWorkerUnavailable
|
|
expr: (sum(up{job="attachment-worker"}) or vector(0)) < 1
|
|
for: 30s
|
|
labels:
|
|
severity: critical
|
|
service: messenger-worker
|
|
annotations:
|
|
summary: No attachment worker is available
|
|
description: Prometheus has not observed a healthy worker target for 30 seconds.
|
|
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
|
|
|
- alert: AttachmentQueueBacklogGrowing
|
|
expr: |
|
|
max(messenger_attachment_jobs_oldest_age_seconds{status="queued"}) > 30
|
|
and max(deriv(messenger_attachment_jobs_oldest_age_seconds{status="queued"}[2m])) > 0
|
|
for: 30s
|
|
labels:
|
|
severity: warning
|
|
service: messenger-worker
|
|
annotations:
|
|
summary: Attachment queue backlog is growing
|
|
description: The oldest queued attachment is older than 30 seconds and its age continues to increase.
|
|
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
|
|
|
- alert: AttachmentRedisStreamLagHigh
|
|
expr: max(redis_stream_group_lag{job="redis-broker",stream="attachment-jobs",group="attachment-workers"}) > 20
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
service: redis-broker
|
|
annotations:
|
|
summary: Redis Stream consumer lag is high
|
|
description: The attachment worker group has more than 20 messages waiting for delivery.
|
|
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|