Files
mtusi2026lab6/evidence/dashboard-validation.json
T

188 lines
7.9 KiB
JSON

{
"captured_at": "2026-11-21T09:17:47.509294Z",
"live_worker_capacity": 1.0,
"panels": 24,
"promql_checks": [
{
"panel": "Business API requests / second",
"query": "sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m]))",
"series": 1
},
{
"panel": "Business API 5xx / all responses",
"query": "(sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\",status_class=\"5xx\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])), 1e-9)",
"series": 1
},
{
"panel": "API mean duration (seconds)",
"query": "sum(rate(messenger_http_duration_seconds_sum{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])) / clamp_min(sum(rate(messenger_http_duration_seconds_count{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])), 1e-9)",
"series": 1
},
{
"panel": "Worker attempt outcomes / second",
"query": "sum by (outcome) (rate(messenger_jobs_processed_total{service_name=\"messenger-worker\"}[1m]))",
"series": 3
},
{
"panel": "Worker mean attempt duration (seconds)",
"query": "sum(rate(messenger_jobs_duration_seconds_sum{service_name=\"messenger-worker\"}[1m])) / clamp_min(sum(rate(messenger_jobs_duration_seconds_count{service_name=\"messenger-worker\"}[1m])), 1e-9)",
"series": 1
},
{
"panel": "Worker completed attempts (counter)",
"query": "sum by (outcome) (messenger_jobs_processed_total{service_name=\"messenger-worker\"})",
"series": 3
},
{
"panel": "Structured application events (expand for trace_id / span_id / request_id)",
"query": "{service_name=~\"messenger-api|messenger-worker|messenger-publisher\"}",
"series": 1343
},
{
"panel": "API duration p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])))",
"series": 1
},
{
"panel": "API duration p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])))",
"series": 1
},
{
"panel": "API duration p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])))",
"series": 1
},
{
"panel": "Worker duration p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.5, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
"series": 1
},
{
"panel": "Worker duration p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.95, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
"series": 1
},
{
"panel": "Worker duration p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.99, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
"series": 1
},
{
"panel": "Durable unfinished jobs (count)",
"query": "max(messenger_queue_depth{service_name=\"messenger-publisher\"})",
"series": 1
},
{
"panel": "Oldest unfinished job / observer sample age (seconds)",
"query": "max(messenger_queue_oldest_age_seconds{service_name=\"messenger-publisher\"})",
"series": 1
},
{
"panel": "Oldest unfinished job / observer sample age (seconds)",
"query": "max(messenger_queue_sample_age_seconds{service_name=\"messenger-publisher\"})",
"series": 1
},
{
"panel": "DB pool used / capacity (connections)",
"query": "messenger_db_pool_used",
"series": 13
},
{
"panel": "DB pool used / capacity (connections)",
"query": "messenger_db_pool_capacity",
"series": 13
},
{
"panel": "DB acquisitions in progress (requests)",
"query": "messenger_db_pool_waiting",
"series": 13
},
{
"panel": "Worker saturation (busy / capacity)",
"query": "(sum(messenger_worker_busy{service_name=\"messenger-worker\"} and on(service_instance_id) (time() - messenger_worker_heartbeat_seconds{service_name=\"messenger-worker\"} < 15)) or vector(0)) / (sum(messenger_worker_capacity{service_name=\"messenger-worker\"} and on(service_instance_id) (time() - messenger_worker_heartbeat_seconds{service_name=\"messenger-worker\"} < 15)) or vector(0))",
"series": 1
},
{
"panel": "Worker busy / capacity (jobs)",
"query": "sum(messenger_worker_busy{service_name=\"messenger-worker\"} and on(service_instance_id) (time() - messenger_worker_heartbeat_seconds{service_name=\"messenger-worker\"} < 15)) or vector(0)",
"series": 1
},
{
"panel": "Worker busy / capacity (jobs)",
"query": "sum(messenger_worker_capacity{service_name=\"messenger-worker\"} and on(service_instance_id) (time() - messenger_worker_heartbeat_seconds{service_name=\"messenger-worker\"} < 15)) or vector(0)",
"series": 1
},
{
"panel": "Application CPU (cores)",
"query": "rate(messenger_process_cpu_seconds_total[1m])",
"series": 13
},
{
"panel": "Application resident RAM (bytes)",
"query": "messenger_process_memory_bytes",
"series": 13
},
{
"panel": "Acceptance to ready p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
"series": 1
},
{
"panel": "Acceptance to ready p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
"series": 1
},
{
"panel": "Acceptance to ready p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
"series": 1
},
{
"panel": "API availability SLI / SLO (5 minutes)",
"query": "1 - (sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\",status_class=\"5xx\"}[5m])) or vector(0)) / sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\"}[5m]))",
"series": 1
},
{
"panel": "API availability SLI / SLO (5 minutes)",
"query": "vector(0.99)",
"series": 1
},
{
"panel": "Ready within 60 seconds SLI / SLO (5 minutes)",
"query": "sum(rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\",le=\"60\"}[5m])) / sum(rate(messenger_attachment_time_to_ready_seconds_count{service_name=\"messenger-worker\"}[5m]))",
"series": 1
},
{
"panel": "Ready within 60 seconds SLI / SLO (5 minutes)",
"query": "vector(0.95)",
"series": 1
},
{
"panel": "Local alert state: pending / firing",
"query": "ALERTS{alertname=~\"MessengerHighErrorRate|MessengerQueueLag\"}",
"series": 0
},
{
"panel": "Collector queue occupancy (batches)",
"query": "otelcol_exporter_queue_size{exporter=~\"otlphttp/.*\"}",
"series": 2
},
{
"panel": "Collector queue occupancy (batches)",
"query": "otelcol_exporter_queue_capacity{exporter=~\"otlphttp/.*\"}",
"series": 2
},
{
"panel": "Collector failed export attempts / second",
"query": "rate(otelcol_exporter_send_failed_spans[1m])",
"series": 1
},
{
"panel": "Collector failed export attempts / second",
"query": "rate(otelcol_exporter_send_failed_log_records[1m])",
"series": 1
}
]
}