2085 lines
59 KiB
JSON
2085 lines
59 KiB
JSON
{
|
|
"started_at": "2026-11-20T19:13:17.601390+00:00",
|
|
"slo": {
|
|
"window": "10m rolling (local exercise)",
|
|
"api": "99% non-5xx of business requests",
|
|
"ready": "95% job versions ready within 5s; cohorts mature at 5s, unfinished count bad"
|
|
},
|
|
"incidents": [
|
|
{
|
|
"name": "worker_stopped",
|
|
"accepted_jobs": 32,
|
|
"all_ready_and_identical": true,
|
|
"detection_seconds": 15.526613235473633,
|
|
"burn_detection_seconds": 19.666218519210815,
|
|
"recovery_observation_seconds": 23.199,
|
|
"timeline": [
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202057.5768352,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202058.601404,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202059.6297228,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202060.66305,
|
|
"alerts": [],
|
|
"queue_depth": 2.0,
|
|
"oldest_seconds": 1.869852,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202061.6982248,
|
|
"alerts": [],
|
|
"queue_depth": 4.0,
|
|
"oldest_seconds": 3.87416,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202062.730138,
|
|
"alerts": [],
|
|
"queue_depth": 4.0,
|
|
"oldest_seconds": 3.87416,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202063.759177,
|
|
"alerts": [],
|
|
"queue_depth": 6.0,
|
|
"oldest_seconds": 5.869632,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202064.788148,
|
|
"alerts": [],
|
|
"queue_depth": 6.0,
|
|
"oldest_seconds": 5.869632,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202065.814267,
|
|
"alerts": [],
|
|
"queue_depth": 8.0,
|
|
"oldest_seconds": 7.869531,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202066.8488162,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 8.0,
|
|
"oldest_seconds": 7.869531,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202067.892648,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 10.0,
|
|
"oldest_seconds": 9.869192,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202068.9259908,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 10.0,
|
|
"oldest_seconds": 9.869192,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202069.9677482,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 12.0,
|
|
"oldest_seconds": 11.869109,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202071.0000288,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 12.0,
|
|
"oldest_seconds": 11.869109,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202072.034443,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 14.0,
|
|
"oldest_seconds": 13.871986,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202073.065943,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 14.0,
|
|
"oldest_seconds": 13.871986,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202074.09918,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 16.0,
|
|
"oldest_seconds": 15.873493,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202075.135117,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 16.0,
|
|
"oldest_seconds": 15.873493,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202076.171317,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 18.0,
|
|
"oldest_seconds": 17.868514,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202077.2055511,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 18.0,
|
|
"oldest_seconds": 17.868514,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202078.246258,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 20.0,
|
|
"oldest_seconds": 19.86765,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202079.2870898,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 20.0,
|
|
"oldest_seconds": 19.86765,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202080.318822,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 22.0,
|
|
"oldest_seconds": 21.870114,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202081.351396,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 22.0,
|
|
"oldest_seconds": 21.870114,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202082.3855228,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 24.0,
|
|
"oldest_seconds": 23.869201,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202083.424837,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 24.0,
|
|
"oldest_seconds": 23.869201,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202084.455771,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 26.0,
|
|
"oldest_seconds": 25.868467,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202085.490881,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 26.0,
|
|
"oldest_seconds": 25.868467,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202086.518718,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 27.0,
|
|
"oldest_seconds": 27.869149,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202087.553177,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 27.0,
|
|
"oldest_seconds": 27.869149,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202088.596426,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 29.0,
|
|
"oldest_seconds": 29.844103,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202089.633395,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 29.0,
|
|
"oldest_seconds": 29.844103,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202091.838895,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 17.0,
|
|
"oldest_seconds": 18.336906,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 1.009,
|
|
"time": 1795202092.847371,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 17.0,
|
|
"oldest_seconds": 18.336906,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 2.024,
|
|
"time": 1795202093.862734,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 3.04,
|
|
"time": 1795202094.8787389,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 4.059,
|
|
"time": 1795202095.8976982,
|
|
"alerts": [
|
|
{
|
|
"name": "QueueLag",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
},
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 5.082,
|
|
"time": 1795202096.9196181,
|
|
"alerts": [
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 6.106,
|
|
"time": 1795202097.9437141,
|
|
"alerts": [
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 7.122,
|
|
"time": 1795202098.960205,
|
|
"alerts": [
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 8.143,
|
|
"time": 1795202099.980282,
|
|
"alerts": [
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 9.166,
|
|
"time": 1795202101.003215,
|
|
"alerts": [
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 10.179,
|
|
"time": 1795202102.016534,
|
|
"alerts": [
|
|
{
|
|
"name": "ReadyBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:14:24.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 11.201,
|
|
"time": 1795202103.0383081,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 12.217,
|
|
"time": 1795202104.0545878,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 13.227,
|
|
"time": 1795202105.064407,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 14.244,
|
|
"time": 1795202106.080757,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 15.267,
|
|
"time": 1795202107.103968,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 16.289,
|
|
"time": 1795202108.1264188,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 17.311,
|
|
"time": 1795202109.148449,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 18.326,
|
|
"time": 1795202110.162765,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 19.34,
|
|
"time": 1795202111.1773472,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 20.352,
|
|
"time": 1795202112.188322,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 21.365,
|
|
"time": 1795202113.202014,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "postgres_chat_table_lock",
|
|
"short_spike_seconds": 3,
|
|
"spike_http_errors": 8,
|
|
"spike_requests": 292,
|
|
"false_fast_burn_firings": 0,
|
|
"sustained_lock_seconds": 45,
|
|
"detection_seconds": 24.329,
|
|
"trace_id": "a02eea4118e6406ca3b2a50f647c3706",
|
|
"slow_db_spans": [
|
|
{
|
|
"service": "messenger-api",
|
|
"name": "db.pool.acquire",
|
|
"id": "8vXMgCVxfRo=",
|
|
"parent_id": "W+a1plu4JDk=",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"status": {},
|
|
"duration_ms": 0.0035
|
|
},
|
|
{
|
|
"service": "messenger-api",
|
|
"name": "db.SELECT",
|
|
"id": "+nX+TBpQfwc=",
|
|
"parent_id": "W+a1plu4JDk=",
|
|
"kind": "SPAN_KIND_CLIENT",
|
|
"status": {},
|
|
"duration_ms": 0.270667
|
|
},
|
|
{
|
|
"service": "messenger-api",
|
|
"name": "db.pool.acquire",
|
|
"id": "Od1SD2h6n/A=",
|
|
"parent_id": "W+a1plu4JDk=",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"status": {},
|
|
"duration_ms": 0.01125
|
|
},
|
|
{
|
|
"service": "messenger-api",
|
|
"name": "db.SELECT",
|
|
"id": "2FtvyJBJ0dQ=",
|
|
"parent_id": "W+a1plu4JDk=",
|
|
"kind": "SPAN_KIND_CLIENT",
|
|
"status": {
|
|
"message": "dependency operation failed",
|
|
"code": "STATUS_CODE_ERROR"
|
|
},
|
|
"duration_ms": 1001.004792
|
|
}
|
|
],
|
|
"timeline": [
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202145.9289339,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 1.014,
|
|
"time": 1795202146.942615,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 2.031,
|
|
"time": 1795202147.959539,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.027197670295323956
|
|
},
|
|
{
|
|
"elapsed": 3.057,
|
|
"time": 1795202148.986024,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.030292413758586597
|
|
},
|
|
{
|
|
"elapsed": 4.071,
|
|
"time": 1795202149.999698,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.06765702209652857
|
|
},
|
|
{
|
|
"elapsed": 5.097,
|
|
"time": 1795202151.025982,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.07493061979648473
|
|
},
|
|
{
|
|
"elapsed": 6.109,
|
|
"time": 1795202152.037726,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.06785980611483967
|
|
},
|
|
{
|
|
"elapsed": 7.125,
|
|
"time": 1795202153.0539808,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.07516589211810132
|
|
},
|
|
{
|
|
"elapsed": 8.142,
|
|
"time": 1795202154.071126,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.069461713299174
|
|
},
|
|
{
|
|
"elapsed": 9.159,
|
|
"time": 1795202155.088319,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.07690373873626158
|
|
},
|
|
{
|
|
"elapsed": 10.175,
|
|
"time": 1795202156.10375,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.03225806451612903
|
|
},
|
|
{
|
|
"elapsed": 11.195,
|
|
"time": 1795202157.12349,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.03225806451612903
|
|
},
|
|
{
|
|
"elapsed": 12.221,
|
|
"time": 1795202158.149801,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 13.238,
|
|
"time": 1795202159.167267,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 14.253,
|
|
"time": 1795202160.182168,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 15.275,
|
|
"time": 1795202161.203445,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 16.298,
|
|
"time": 1795202162.227231,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 17.315,
|
|
"time": 1795202163.243519,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202164.2827241,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202166.3054059,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202168.334114,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.08490566037735849
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202170.351646,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.24
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202172.373075,
|
|
"alerts": [
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.5227272727272727
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202174.411339,
|
|
"alerts": [
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202176.435264,
|
|
"alerts": [
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202178.4647832,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202180.487332,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202182.515401,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202184.538444,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202186.5624409,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202188.591359,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "pending",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202190.620266,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202192.65135,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202194.676002,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202196.696597,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202198.723835,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202200.744637,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202202.768477,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202204.786269,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202206.804723,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202208.823174,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 0.0,
|
|
"time": 1795202209.835613,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 1.011,
|
|
"time": 1795202210.846758,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 1.0
|
|
},
|
|
{
|
|
"elapsed": 2.023,
|
|
"time": 1795202211.8588831,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.3898305084745763
|
|
},
|
|
{
|
|
"elapsed": 3.04,
|
|
"time": 1795202212.875914,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.3898305084745763
|
|
},
|
|
{
|
|
"elapsed": 4.056,
|
|
"time": 1795202213.892092,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.2
|
|
},
|
|
{
|
|
"elapsed": 5.074,
|
|
"time": 1795202214.910124,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.2
|
|
},
|
|
{
|
|
"elapsed": 6.088,
|
|
"time": 1795202215.9236732,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.07438016528925619
|
|
},
|
|
{
|
|
"elapsed": 7.109,
|
|
"time": 1795202216.944339,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.07438016528925619
|
|
},
|
|
{
|
|
"elapsed": 8.128,
|
|
"time": 1795202217.96331,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 9.143,
|
|
"time": 1795202218.978794,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 10.162,
|
|
"time": 1795202219.998059,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetFastBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:16.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 11.188,
|
|
"time": 1795202221.023726,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 12.212,
|
|
"time": 1795202222.0478778,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 13.231,
|
|
"time": 1795202223.066495,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 14.248,
|
|
"time": 1795202224.0837739,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 15.272,
|
|
"time": 1795202225.1077938,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 16.297,
|
|
"time": 1795202226.132202,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 17.317,
|
|
"time": 1795202227.152969,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 18.333,
|
|
"time": 1795202228.1689012,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 19.353,
|
|
"time": 1795202229.1883461,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 20.371,
|
|
"time": 1795202230.206249,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 21.389,
|
|
"time": 1795202231.225141,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 22.407,
|
|
"time": 1795202232.243118,
|
|
"alerts": [
|
|
{
|
|
"name": "APIHighErrorRate",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:14.478517494Z"
|
|
},
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 23.429,
|
|
"time": 1795202233.264371,
|
|
"alerts": [
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 24.439,
|
|
"time": 1795202234.274912,
|
|
"alerts": [
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 25.455,
|
|
"time": 1795202235.290634,
|
|
"alerts": [
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 26.474,
|
|
"time": 1795202236.309878,
|
|
"alerts": [
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 27.488,
|
|
"time": 1795202237.3232498,
|
|
"alerts": [
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 28.507,
|
|
"time": 1795202238.3424609,
|
|
"alerts": [
|
|
{
|
|
"name": "APIBudgetSlowBurn",
|
|
"state": "firing",
|
|
"active_at": "2026-11-20T19:16:08.478517494Z"
|
|
}
|
|
],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 29.524,
|
|
"time": 1795202239.359757,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 30.546,
|
|
"time": 1795202240.3819351,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
},
|
|
{
|
|
"elapsed": 31.563,
|
|
"time": 1795202241.398821,
|
|
"alerts": [],
|
|
"queue_depth": 0.0,
|
|
"oldest_seconds": 0.0,
|
|
"api_error_ratio": 0.0
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "telemetry_backends_stopped",
|
|
"successful_messages": 1908,
|
|
"business_errors": 0,
|
|
"failed_span_exports_delta": 1323.0,
|
|
"collector_memory_samples": [
|
|
"mtusi-lab6-kraykin-collector-1 70.15MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 70.21MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 70.21MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 67.77MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 67.78MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 67.92MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 67.92MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 67.93MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 67.93MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 66.48MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 66.25MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 66.25MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 66.25MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 66.25MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 68.35MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 70.25MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 71.31MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 71.05MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 71.08MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 71.07MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 71.09MiB / 256MiB",
|
|
"mtusi-lab6-kraykin-collector-1 71.08MiB / 256MiB"
|
|
],
|
|
"queue_bound_batches": 64,
|
|
"export_retry_budget_seconds": 15
|
|
}
|
|
],
|
|
"finished_at": "2026-11-20T19:19:03.135680+00:00"
|
|
}
|