forked from mtusi2026/mtusi2026lab6
174 lines
5.6 KiB
JSON
174 lines
5.6 KiB
JSON
{
|
|
"seconds_to_firing": 30.1,
|
|
"oldest_job_age": [
|
|
{
|
|
"metric": {
|
|
"__name__": "messenger_queue_oldest_age_seconds",
|
|
"deployment_environment_name": "lab",
|
|
"exported_instance": "af26b1841208",
|
|
"exported_job": "messenger-publisher",
|
|
"instance": "otel-collector:8889",
|
|
"job": "application",
|
|
"queue": "attachment.process.v1",
|
|
"service_instance_id": "af26b1841208",
|
|
"service_name": "messenger-publisher",
|
|
"service_version": "local",
|
|
"telemetry_sdk_language": "python",
|
|
"telemetry_sdk_name": "opentelemetry",
|
|
"telemetry_sdk_version": "1.38.0"
|
|
},
|
|
"value": [
|
|
1795196321.26,
|
|
"0"
|
|
]
|
|
},
|
|
{
|
|
"metric": {
|
|
"__name__": "messenger_queue_oldest_age_seconds",
|
|
"deployment_environment_name": "lab",
|
|
"exported_instance": "459bbaf48a82",
|
|
"exported_job": "messenger-publisher",
|
|
"instance": "otel-collector:8889",
|
|
"job": "application",
|
|
"queue": "attachment.process.v1",
|
|
"service_instance_id": "459bbaf48a82",
|
|
"service_name": "messenger-publisher",
|
|
"service_version": "local",
|
|
"telemetry_sdk_language": "python",
|
|
"telemetry_sdk_name": "opentelemetry",
|
|
"telemetry_sdk_version": "1.38.0"
|
|
},
|
|
"value": [
|
|
1795196321.26,
|
|
"25.465688"
|
|
]
|
|
}
|
|
],
|
|
"backlog": [
|
|
{
|
|
"metric": {
|
|
"__name__": "messenger_queue_depth",
|
|
"deployment_environment_name": "lab",
|
|
"exported_instance": "af26b1841208",
|
|
"exported_job": "messenger-publisher",
|
|
"instance": "otel-collector:8889",
|
|
"job": "application",
|
|
"queue": "attachment.process.v1",
|
|
"service_instance_id": "af26b1841208",
|
|
"service_name": "messenger-publisher",
|
|
"service_version": "local",
|
|
"telemetry_sdk_language": "python",
|
|
"telemetry_sdk_name": "opentelemetry",
|
|
"telemetry_sdk_version": "1.38.0"
|
|
},
|
|
"value": [
|
|
1795196321.262,
|
|
"0"
|
|
]
|
|
},
|
|
{
|
|
"metric": {
|
|
"__name__": "messenger_queue_depth",
|
|
"deployment_environment_name": "lab",
|
|
"exported_instance": "459bbaf48a82",
|
|
"exported_job": "messenger-publisher",
|
|
"instance": "otel-collector:8889",
|
|
"job": "application",
|
|
"queue": "attachment.process.v1",
|
|
"service_instance_id": "459bbaf48a82",
|
|
"service_name": "messenger-publisher",
|
|
"service_version": "local",
|
|
"telemetry_sdk_language": "python",
|
|
"telemetry_sdk_name": "opentelemetry",
|
|
"telemetry_sdk_version": "1.38.0"
|
|
},
|
|
"value": [
|
|
1795196321.262,
|
|
"1"
|
|
]
|
|
}
|
|
],
|
|
"alerts_firing": [
|
|
{
|
|
"labels": {
|
|
"alertname": "QueueLagGrowing",
|
|
"deployment_environment_name": "lab",
|
|
"exported_instance": "459bbaf48a82",
|
|
"exported_job": "messenger-publisher",
|
|
"instance": "otel-collector:8889",
|
|
"job": "application",
|
|
"queue": "attachment.process.v1",
|
|
"service_instance_id": "459bbaf48a82",
|
|
"service_name": "messenger-publisher",
|
|
"service_version": "local",
|
|
"severity": "warning",
|
|
"telemetry_sdk_language": "python",
|
|
"telemetry_sdk_name": "opentelemetry",
|
|
"telemetry_sdk_version": "1.38.0"
|
|
},
|
|
"annotations": {
|
|
"description": "Inspect worker capacity, busy gauge and consumer traces. Restore worker and verify backlog drains. See infra/runbook.md.",
|
|
"summary": "Oldest unfinished job exceeds 15 seconds"
|
|
},
|
|
"state": "firing",
|
|
"activeAt": "2026-11-20T17:38:31.478517494Z",
|
|
"value": "2.2414185e+01"
|
|
},
|
|
{
|
|
"labels": {
|
|
"alertname": "QueueObservationStale",
|
|
"deployment_environment_name": "lab",
|
|
"exported_instance": "af26b1841208",
|
|
"exported_job": "messenger-publisher",
|
|
"instance": "otel-collector:8889",
|
|
"job": "application",
|
|
"queue": "attachment.process.v1",
|
|
"service_instance_id": "af26b1841208",
|
|
"service_name": "messenger-publisher",
|
|
"service_version": "local",
|
|
"severity": "warning",
|
|
"telemetry_sdk_language": "python",
|
|
"telemetry_sdk_name": "opentelemetry",
|
|
"telemetry_sdk_version": "1.38.0"
|
|
},
|
|
"annotations": {
|
|
"description": "Check publisher and PostgreSQL availability before trusting queue gauges.",
|
|
"summary": "Publisher backlog observations are stale"
|
|
},
|
|
"state": "firing",
|
|
"activeAt": "2026-11-20T17:38:01.478517494Z",
|
|
"value": "5.170885992050171e+01"
|
|
}
|
|
],
|
|
"resolved": true,
|
|
"trace_id": "a0d4df4afbe44a30999830b9d563817c",
|
|
"span_count": 16,
|
|
"alerts_after": [
|
|
{
|
|
"labels": {
|
|
"alertname": "QueueObservationStale",
|
|
"deployment_environment_name": "lab",
|
|
"exported_instance": "af26b1841208",
|
|
"exported_job": "messenger-publisher",
|
|
"instance": "otel-collector:8889",
|
|
"job": "application",
|
|
"queue": "attachment.process.v1",
|
|
"service_instance_id": "af26b1841208",
|
|
"service_name": "messenger-publisher",
|
|
"service_version": "local",
|
|
"severity": "warning",
|
|
"telemetry_sdk_language": "python",
|
|
"telemetry_sdk_name": "opentelemetry",
|
|
"telemetry_sdk_version": "1.38.0"
|
|
},
|
|
"annotations": {
|
|
"description": "Check publisher and PostgreSQL availability before trusting queue gauges.",
|
|
"summary": "Publisher backlog observations are stale"
|
|
},
|
|
"state": "firing",
|
|
"activeAt": "2026-11-20T17:38:01.478517494Z",
|
|
"value": "6.370885992050171e+01"
|
|
}
|
|
]
|
|
}
|