UTC 2026-11-05T22:40:10Z
HEAD 639582280f0cc2236dee3140207168df23cc8bf1
$ le values
{"status":"success","data":["+Inf","1.0","10.0","120.0","2.5","20.0","30.0","300.0","5.0","60.0"]}
$ rule groups
{"status":"success","data":{"groups":[{"name":"lab6-alerts","file":"/etc/prometheus/rules/alerts.yml","rules":[{"state":"firing","name":"APIErrorBudgetFastBurn","query":"slo:api_error_ratio:1h \u003e 0.072 and slo:api_error_ratio:5m \u003e 0.072 and on () sum(increase(haproxy_backend_http_responses_total{proxy=\"api_backend\"}[5m])) \u003e= 20","duration":0,"keepFiringFor":0,"labels":{"severity":"page","slo":"api-availability"},"annotations":{"runbook":"infra/runbooks/slo-burn.md","summary":"API 5xx burn rate is above 14.4x"},"alerts":[{"labels":{"alertname":"APIErrorBudgetFastBurn","severity":"page","slo":"api-availability"},"annotations":{"runbook":"infra/runbooks/slo-burn.md","summary":"API 5xx burn rate is above 14.4x"},"state":"firing","activeAt":"2026-11-05T22:37:18.187213033Z","value":"4.236111111111111e-01"}],"health":"ok","evaluationTime":0.000478281,"lastEvaluation":"2026-11-05T22:40:06.465725113Z","type":"alerting"},{"state":"firing","name":"APIErrorBudgetSlowBurn","query":"slo:api_error_ratio:6h \u003e 0.03 and slo:api_error_ratio:30m \u003e 0.03 and on () sum(increase(haproxy_backend_http_responses_total{proxy=\"api_backend\"}[30m])) \u003e= 100","duration":0,"keepFiringFor":0,"labels":{"severity":"ticket","slo":"api-availability"},"annotations":{"runbook":"infra/runbooks/slo-burn.md","summary":"API 5xx burn rate is above 6x"},"alerts":[{"labels":{"alertname":"APIErrorBudgetSlowBurn","severity":"ticket","slo":"api-availability"},"annotations":{"runbook":"infra/runbooks/slo-burn.md","summary":"API 5xx burn rate is above 6x"},"state":"firing","activeAt":"2026-11-05T22:38:08.187213033Z","value":"4.236111111111111e-01"}],"health":"ok","evaluationTime":0.000176797,"lastEvaluation":"2026-11-05T22:40:06.466208397Z","type":"alerting"},{"state":"inactive","name":"TTRErrorBudgetFastBurn","query":"slo:job_ttr_bad_ratio:1h \u003e 0.144 and slo:job_ttr_bad_ratio:5m \u003e 0.144 and on () sum(increase(messenger_job_time_to_ready_seconds_count[5m])) + sum(increase(messenger_jobs_ttr_missed_total[5m])) \u003e= 10","duration":0,"keepFiringFor":0,"labels":{"severity":"page","slo":"time-to-ready"},"annotations":{"runbook":"infra/runbooks/queue-lag.md","summary":"Jobs are missing the 30s time-to-ready objective"},"alerts":[],"health":"ok","evaluationTime":0.00011289,"lastEvaluation":"2026-11-05T22:40:06.466387903Z","type":"alerting"},{"state":"inactive","name":"TTRErrorBudgetSlowBurn","query":"slo:job_ttr_bad_ratio:6h \u003e 0.06 and slo:job_ttr_bad_ratio:30m \u003e 0.06 and on () sum(increase(messenger_job_time_to_ready_seconds_count[30m])) + sum(increase(messenger_jobs_ttr_missed_total[30m])) \u003e= 30","duration":0,"keepFiringFor":0,"labels":{"severity":"ticket","slo":"time-to-ready"},"annotations":{"runbook":"infra/runbooks/queue-lag.md","summary":"Time-to-ready budget is burning slowly"},"alerts":[],"health":"ok","evaluationTime":0.000066325,"lastEvaluation":"2026-11-05T22:40:06.466503127Z","type":"alerting"},{"state":"inactive","name":"QueueOldestJobAgeHigh","query":"max(messenger_queue_oldest_job_age_seconds) \u003e 120","duration":600,"keepFiringFor":0,"labels":{"severity":"warning"},"annotations":{"runbook":"infra/runbooks/queue-lag.md","summary":"The oldest queued/processing job is older than 2 minutes"},"alerts":[],"health":"ok","evaluationTime":0.000030557,"lastEvaluation":"2026-11-05T22:40:06.466571245Z","type":"alerting"},{"state":"inactive","name":"QueueOldestJobAgeCritical","query":"max(messenger_queue_oldest_job_age_seconds) \u003e 600","duration":300,"keepFiringFor":0,"labels":{"severity":"page"},"annotations":{"runbook":"infra/runbooks/queue-lag.md","summary":"The oldest queued/processing job is older than 10 minutes"},"alerts":[],"health":"ok","evaluationTime":0.000023971,"lastEvaluation":"2026-11-05T22:40:06.466603552Z","type":"alerting"},{"state":"inactive","name":"QueueDepthGrowing","query":"max(rabbitmq_queue_messages_ready{queue=\"attachment.process.v1\"}) \u003e 20 and on () max(deriv(rabbitmq_queue_messages_ready{queue=\"attachment.process.v1\"}[10m])) \u003e 0","duration":600,"keepFiringFor":0,"labels":{"severity":"warning"},"annotations":{"runbook":"infra/runbooks/queue-lag.md","summary":"RabbitMQ ready depth is growing"},"alerts":[],"health":"ok","evaluationTime":0.000058112,"lastEvaluation":"2026-11-05T22:40:06.466629232Z","type":"alerting"},{"state":"inactive","name":"JobsOverdue","query":"max(messenger_jobs_overdue) \u003e 0","duration":120,"keepFiringFor":0,"labels":{"severity":"warning","slo":"time-to-ready"},"annotations":{"runbook":"infra/runbooks/queue-lag.md","summary":"Jobs have been queued/processing for more than 30 seconds"},"alerts":[],"health":"ok","evaluationTime":0.000025346,"lastEvaluation":"2026-11-05T22:40:06.46668922Z","type":"alerting"},{"state":"inactive","name":"WorkerSaturated","query":"(sum(messenger_worker_busy) / clamp_min(sum(messenger_worker_capacity), 1) \u003e 0.8) and (max(messenger_queue_oldest_job_age_seconds) \u003e 120)","duration":600,"keepFiringFor":0,"labels":{"severity":"warning"},"annotations":{"runbook":"infra/runbooks/queue-lag.md","summary":"Worker slots are saturated while the queue is aging"},"alerts":[],"health":"ok","evaluationTime":0.000075204,"lastEvaluation":"2026-11-05T22:40:06.466716067Z","type":"alerting"},{"state":"inactive","name":"DBPoolSaturated","query":"((sum(rate(messenger_db_pool_empty_acquire_wait_seconds_total[5m])) / clamp_min(sum(rate(messenger_db_pool_empty_acquire_total[5m])), 0.000000001) \u003e 0.05) and (sum(rate(messenger_db_pool_empty_acquire_total[5m])) \u003e 0.1)) or (sum(messenger_db_pool_used) / clamp_min(sum(messenger_db_pool_max), 1) \u003e 0.8)","duration":300,"keepFiringFor":0,"labels":{"severity":"warning"},"annotations":{"runbook":"infra/runbooks/db-availability.md","summary":"PostgreSQL pool is waiting or nearly exhausted"},"alerts":[],"health":"ok","evaluationTime":0.000168042,"lastEvaluation":"2026-11-05T22:40:06.466793439Z","type":"alerting"},{"state":"inactive","name":"CollectorDroppedTelemetry","query":"sum(rate(otelcol_receiver_failed_spans[5m])) \u003e 0 or sum(rate(otelcol_receiver_failed_metric_points[5m])) \u003e 0 or sum(rate(otelcol_receiver_failed_log_records[5m])) \u003e 0 or sum(rate(otelcol_receiver_refused_spans[5m])) \u003e 0 or sum(rate(otelcol_receiver_refused_metric_points[5m])) \u003e 0 or sum(rate(otelcol_receiver_refused_log_records[5m])) \u003e 0 or sum(otelcol_exporter_queue_size) \u003e= sum(otelcol_exporter_queue_capacity) * 0.9","duration":300,"keepFiringFor":0,"labels":{"severity":"warning"},"annotations":{"runbook":"infra/runbooks/telemetry-backend-down.md","summary":"Collector is dropping or refusing telemetry"},"alerts":[],"health":"ok","evaluationTime":0.000185801,"lastEvaluation":"2026-11-05T22:40:06.466963691Z","type":"alerting"},{"state":"inactive","name":"TelemetryStorageDown","query":"up{job=~\"loki|tempo\"} == 0","duration":120,"keepFiringFor":0,"labels":{"component":"telemetry","severity":"warning"},"annotations":{"runbook":"infra/runbooks/telemetry-backend-down.md","summary":"Loki or Tempo is unavailable; Collector queues are bounded"},"alerts":[],"health":"ok","evaluationTime":0.000044439,"lastEvaluation":"2026-11-05T22:40:06.467151493Z","type":"alerting"},{"state":"inactive","name":"TelemetryCollectorDown","query":"up{job=\"otel-collector\"} == 0","duration":120,"keepFiringFor":0,"labels":{"component":"telemetry","severity":"warning"},"annotations":{"runbook":"infra/runbooks/telemetry-backend-down.md","summary":"OTel Collector is unavailable; application keeps running"},"alerts":[],"health":"ok","evaluationTime":0.00002989,"lastEvaluation":"2026-11-05T22:40:06.467197724Z","type":"alerting"},{"state":"inactive","name":"APISLIMissing","query":"absent(slo:api_error_ratio:5m) or up{job=\"haproxy\"} == 0","duration":120,"keepFiringFor":0,"labels":{"severity":"page"},"annotations":{"runbook":"infra/runbooks/api-errors.md","summary":"Edge availability data is missing"},"alerts":[],"health":"ok","evaluationTime":0.000041354,"lastEvaluation":"2026-11-05T22:40:06.46722974Z","type":"alerting"},{"state":"inactive","name":"QueueMetricsStale","query":"(absent(messenger_queue_oldest_job_age_seconds) or absent(messenger_jobs_accepted_total)) and on () (up{job=\"otel-collector\"} == 1)","duration":120,"keepFiringFor":0,"labels":{"severity":"warning"},"annotations":{"runbook":"infra/runbooks/queue-lag.md","summary":"Queue/backlog metrics are missing"},"alerts":[],"health":"ok","evaluationTime":0.000058112,"lastEvaluation":"2026-11-05T22:40:06.467273762Z","type":"alerting"}],"interval":5,"limit":0,"evaluationTime":0.001646906,"lastEvaluation":"2026-11-05T22:40:06.46568747Z"},{"name":"slo","file":"/etc/prometheus/rules/slo.yml","rules":[{"name":"slo:api_error_ratio:5m","query":"sum(rate(haproxy_backend_http_responses_total{code=~\"5..\",proxy=\"api_backend\"}[5m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy=\"api_backend\"}[5m])), 0.001)","labels":{},"health":"ok","evaluationTime":0.000303693,"lastEvaluation":"2026-11-05T22:40:06.173494571Z","type":"recording"},{"name":"slo:api_error_ratio:1h","query":"sum(rate(haproxy_backend_http_responses_total{code=~\"5..\",proxy=\"api_backend\"}[1h])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy=\"api_backend\"}[1h])), 0.001)","labels":{},"health":"ok","evaluationTime":0.000135568,"lastEvaluation":"2026-11-05T22:40:06.1738031Z","type":"recording"},{"name":"slo:api_error_ratio:6h","query":"sum(rate(haproxy_backend_http_responses_total{code=~\"5..\",proxy=\"api_backend\"}[6h])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy=\"api_backend\"}[6h])), 0.001)","labels":{},"health":"ok","evaluationTime":0.000105427,"lastEvaluation":"2026-11-05T22:40:06.17394267Z","type":"recording"},{"name":"slo:api_error_ratio:30m","query":"sum(rate(haproxy_backend_http_responses_total{code=~\"5..\",proxy=\"api_backend\"}[30m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy=\"api_backend\"}[30m])), 0.001)","labels":{},"health":"ok","evaluationTime":0.000143071,"lastEvaluation":"2026-11-05T22:40:06.174051808Z","type":"recording"},{"name":"slo:job_ttr_bad_ratio:5m","query":"sum(rate(messenger_jobs_ttr_missed_total[5m])) / clamp_min(sum(rate(messenger_job_time_to_ready_seconds_bucket{le=\"30.0\"}[5m])) + sum(rate(messenger_jobs_ttr_missed_total[5m])), 0.001) and (sum(increase(messenger_job_time_to_ready_seconds_count[5m])) + sum(increase(messenger_jobs_ttr_missed_total[5m]))) \u003e= 10","labels":{},"health":"ok","evaluationTime":0.000131982,"lastEvaluation":"2026-11-05T22:40:06.174197464Z","type":"recording"},{"name":"slo:job_ttr_bad_ratio:1h","query":"sum(rate(messenger_jobs_ttr_missed_total[1h])) / clamp_min(sum(rate(messenger_job_time_to_ready_seconds_bucket{le=\"30.0\"}[1h])) + sum(rate(messenger_jobs_ttr_missed_total[1h])), 0.001) and (sum(increase(messenger_job_time_to_ready_seconds_count[1h])) + sum(increase(messenger_jobs_ttr_missed_total[1h]))) \u003e= 10","labels":{},"health":"ok","evaluationTime":0.000168335,"lastEvaluation":"2026-11-05T22:40:06.174332365Z","type":"recording"},{"name":"slo:job_ttr_bad_ratio:6h","query":"sum(rate(messenger_jobs_ttr_missed_total[6h])) / clamp_min(sum(rate(messenger_job_time_to_ready_seconds_bucket{le=\"30.0\"}[6h])) + sum(rate(messenger_jobs_ttr_missed_total[6h])), 0.001) and (sum(increase(messenger_job_time_to_ready_seconds_count[6h])) + sum(increase(messenger_jobs_ttr_missed_total[6h]))) \u003e= 30","labels":{},"health":"ok","evaluationTime":0.000134526,"lastEvaluation":"2026-11-05T22:40:06.174503825Z","type":"recording"},{"name":"slo:job_ttr_bad_ratio:30m","query":"sum(rate(messenger_jobs_ttr_missed_total[30m])) / clamp_min(sum(rate(messenger_job_time_to_ready_seconds_bucket{le=\"30.0\"}[30m])) + sum(rate(messenger_jobs_ttr_missed_total[30m])), 0.001) and (sum(increase(messenger_job_time_to_ready_seconds_count[30m])) + sum(increase(messenger_jobs_ttr_missed_total[30m]))) \u003e= 30","labels":{},"health":"ok","evaluationTime":0.000105302,"lastEvaluation":"2026-11-05T22:40:06.174640602Z","type":"recording"}],"interval":5,"limit":0,"evaluationTime":0.001286977,"lastEvaluation":"2026-11-05T22:40:06.173461971Z"},{"name":"slo-long","file":"/etc/prometheus/rules/slo.yml","rules":[{"name":"slo:api_error_ratio:7d","query":"sum(rate(haproxy_backend_http_responses_total{code=~\"5..\",proxy=\"api_backend\"}[1w])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy=\"api_backend\"}[1w])), 0.001)","labels":{},"health":"ok","evaluationTime":0.000370647,"lastEvaluation":"2026-11-05T22:39:50.739434913Z","type":"recording"},{"name":"slo:api_budget_remaining:7d","query":"clamp(1 - (slo:api_error_ratio:7d / 0.005), 0, 1)","labels":{},"health":"ok","evaluationTime":0.000064142,"lastEvaluation":"2026-11-05T22:39:50.739811034Z","type":"recording"}],"interval":60,"limit":0,"evaluationTime":0.000491703,"lastEvaluation":"2026-11-05T22:39:50.739385396Z"}]}}