188 lines
7.9 KiB
JSON
188 lines
7.9 KiB
JSON
{
|
|
"captured_at": "2026-11-21T09:17:47.509294Z",
|
|
"live_worker_capacity": 1.0,
|
|
"panels": 24,
|
|
"promql_checks": [
|
|
{
|
|
"panel": "Business API requests / second",
|
|
"query": "sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m]))",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Business API 5xx / all responses",
|
|
"query": "(sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\",status_class=\"5xx\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])), 1e-9)",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "API mean duration (seconds)",
|
|
"query": "sum(rate(messenger_http_duration_seconds_sum{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])) / clamp_min(sum(rate(messenger_http_duration_seconds_count{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])), 1e-9)",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Worker attempt outcomes / second",
|
|
"query": "sum by (outcome) (rate(messenger_jobs_processed_total{service_name=\"messenger-worker\"}[1m]))",
|
|
"series": 3
|
|
},
|
|
{
|
|
"panel": "Worker mean attempt duration (seconds)",
|
|
"query": "sum(rate(messenger_jobs_duration_seconds_sum{service_name=\"messenger-worker\"}[1m])) / clamp_min(sum(rate(messenger_jobs_duration_seconds_count{service_name=\"messenger-worker\"}[1m])), 1e-9)",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Worker completed attempts (counter)",
|
|
"query": "sum by (outcome) (messenger_jobs_processed_total{service_name=\"messenger-worker\"})",
|
|
"series": 3
|
|
},
|
|
{
|
|
"panel": "Structured application events (expand for trace_id / span_id / request_id)",
|
|
"query": "{service_name=~\"messenger-api|messenger-worker|messenger-publisher\"}",
|
|
"series": 1343
|
|
},
|
|
{
|
|
"panel": "API duration p50 / p95 / p99 (seconds)",
|
|
"query": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])))",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "API duration p50 / p95 / p99 (seconds)",
|
|
"query": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])))",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "API duration p50 / p95 / p99 (seconds)",
|
|
"query": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])))",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Worker duration p50 / p95 / p99 (seconds)",
|
|
"query": "histogram_quantile(0.5, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Worker duration p50 / p95 / p99 (seconds)",
|
|
"query": "histogram_quantile(0.95, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Worker duration p50 / p95 / p99 (seconds)",
|
|
"query": "histogram_quantile(0.99, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Durable unfinished jobs (count)",
|
|
"query": "max(messenger_queue_depth{service_name=\"messenger-publisher\"})",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Oldest unfinished job / observer sample age (seconds)",
|
|
"query": "max(messenger_queue_oldest_age_seconds{service_name=\"messenger-publisher\"})",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Oldest unfinished job / observer sample age (seconds)",
|
|
"query": "max(messenger_queue_sample_age_seconds{service_name=\"messenger-publisher\"})",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "DB pool used / capacity (connections)",
|
|
"query": "messenger_db_pool_used",
|
|
"series": 13
|
|
},
|
|
{
|
|
"panel": "DB pool used / capacity (connections)",
|
|
"query": "messenger_db_pool_capacity",
|
|
"series": 13
|
|
},
|
|
{
|
|
"panel": "DB acquisitions in progress (requests)",
|
|
"query": "messenger_db_pool_waiting",
|
|
"series": 13
|
|
},
|
|
{
|
|
"panel": "Worker saturation (busy / capacity)",
|
|
"query": "(sum(messenger_worker_busy{service_name=\"messenger-worker\"} and on(service_instance_id) (time() - messenger_worker_heartbeat_seconds{service_name=\"messenger-worker\"} < 15)) or vector(0)) / (sum(messenger_worker_capacity{service_name=\"messenger-worker\"} and on(service_instance_id) (time() - messenger_worker_heartbeat_seconds{service_name=\"messenger-worker\"} < 15)) or vector(0))",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Worker busy / capacity (jobs)",
|
|
"query": "sum(messenger_worker_busy{service_name=\"messenger-worker\"} and on(service_instance_id) (time() - messenger_worker_heartbeat_seconds{service_name=\"messenger-worker\"} < 15)) or vector(0)",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Worker busy / capacity (jobs)",
|
|
"query": "sum(messenger_worker_capacity{service_name=\"messenger-worker\"} and on(service_instance_id) (time() - messenger_worker_heartbeat_seconds{service_name=\"messenger-worker\"} < 15)) or vector(0)",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Application CPU (cores)",
|
|
"query": "rate(messenger_process_cpu_seconds_total[1m])",
|
|
"series": 13
|
|
},
|
|
{
|
|
"panel": "Application resident RAM (bytes)",
|
|
"query": "messenger_process_memory_bytes",
|
|
"series": 13
|
|
},
|
|
{
|
|
"panel": "Acceptance to ready p50 / p95 / p99 (seconds)",
|
|
"query": "histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Acceptance to ready p50 / p95 / p99 (seconds)",
|
|
"query": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Acceptance to ready p50 / p95 / p99 (seconds)",
|
|
"query": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "API availability SLI / SLO (5 minutes)",
|
|
"query": "1 - (sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\",status_class=\"5xx\"}[5m])) or vector(0)) / sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\"}[5m]))",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "API availability SLI / SLO (5 minutes)",
|
|
"query": "vector(0.99)",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Ready within 60 seconds SLI / SLO (5 minutes)",
|
|
"query": "sum(rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\",le=\"60\"}[5m])) / sum(rate(messenger_attachment_time_to_ready_seconds_count{service_name=\"messenger-worker\"}[5m]))",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Ready within 60 seconds SLI / SLO (5 minutes)",
|
|
"query": "vector(0.95)",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Local alert state: pending / firing",
|
|
"query": "ALERTS{alertname=~\"MessengerHighErrorRate|MessengerQueueLag\"}",
|
|
"series": 0
|
|
},
|
|
{
|
|
"panel": "Collector queue occupancy (batches)",
|
|
"query": "otelcol_exporter_queue_size{exporter=~\"otlphttp/.*\"}",
|
|
"series": 2
|
|
},
|
|
{
|
|
"panel": "Collector queue occupancy (batches)",
|
|
"query": "otelcol_exporter_queue_capacity{exporter=~\"otlphttp/.*\"}",
|
|
"series": 2
|
|
},
|
|
{
|
|
"panel": "Collector failed export attempts / second",
|
|
"query": "rate(otelcol_exporter_send_failed_spans[1m])",
|
|
"series": 1
|
|
},
|
|
{
|
|
"panel": "Collector failed export attempts / second",
|
|
"query": "rate(otelcol_exporter_send_failed_log_records[1m])",
|
|
"series": 1
|
|
}
|
|
]
|
|
}
|