Files
mtusi2026lab6/evidence/observability-worker-stop.json
T

174 lines
5.6 KiB
JSON

{
"seconds_to_firing": 30.1,
"oldest_job_age": [
{
"metric": {
"__name__": "messenger_queue_oldest_age_seconds",
"deployment_environment_name": "lab",
"exported_instance": "af26b1841208",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "application",
"queue": "attachment.process.v1",
"service_instance_id": "af26b1841208",
"service_name": "messenger-publisher",
"service_version": "local",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.38.0"
},
"value": [
1795196321.26,
"0"
]
},
{
"metric": {
"__name__": "messenger_queue_oldest_age_seconds",
"deployment_environment_name": "lab",
"exported_instance": "459bbaf48a82",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "application",
"queue": "attachment.process.v1",
"service_instance_id": "459bbaf48a82",
"service_name": "messenger-publisher",
"service_version": "local",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.38.0"
},
"value": [
1795196321.26,
"25.465688"
]
}
],
"backlog": [
{
"metric": {
"__name__": "messenger_queue_depth",
"deployment_environment_name": "lab",
"exported_instance": "af26b1841208",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "application",
"queue": "attachment.process.v1",
"service_instance_id": "af26b1841208",
"service_name": "messenger-publisher",
"service_version": "local",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.38.0"
},
"value": [
1795196321.262,
"0"
]
},
{
"metric": {
"__name__": "messenger_queue_depth",
"deployment_environment_name": "lab",
"exported_instance": "459bbaf48a82",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "application",
"queue": "attachment.process.v1",
"service_instance_id": "459bbaf48a82",
"service_name": "messenger-publisher",
"service_version": "local",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.38.0"
},
"value": [
1795196321.262,
"1"
]
}
],
"alerts_firing": [
{
"labels": {
"alertname": "QueueLagGrowing",
"deployment_environment_name": "lab",
"exported_instance": "459bbaf48a82",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "application",
"queue": "attachment.process.v1",
"service_instance_id": "459bbaf48a82",
"service_name": "messenger-publisher",
"service_version": "local",
"severity": "warning",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.38.0"
},
"annotations": {
"description": "Inspect worker capacity, busy gauge and consumer traces. Restore worker and verify backlog drains. See infra/runbook.md.",
"summary": "Oldest unfinished job exceeds 15 seconds"
},
"state": "firing",
"activeAt": "2026-11-20T17:38:31.478517494Z",
"value": "2.2414185e+01"
},
{
"labels": {
"alertname": "QueueObservationStale",
"deployment_environment_name": "lab",
"exported_instance": "af26b1841208",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "application",
"queue": "attachment.process.v1",
"service_instance_id": "af26b1841208",
"service_name": "messenger-publisher",
"service_version": "local",
"severity": "warning",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.38.0"
},
"annotations": {
"description": "Check publisher and PostgreSQL availability before trusting queue gauges.",
"summary": "Publisher backlog observations are stale"
},
"state": "firing",
"activeAt": "2026-11-20T17:38:01.478517494Z",
"value": "5.170885992050171e+01"
}
],
"resolved": true,
"trace_id": "a0d4df4afbe44a30999830b9d563817c",
"span_count": 16,
"alerts_after": [
{
"labels": {
"alertname": "QueueObservationStale",
"deployment_environment_name": "lab",
"exported_instance": "af26b1841208",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "application",
"queue": "attachment.process.v1",
"service_instance_id": "af26b1841208",
"service_name": "messenger-publisher",
"service_version": "local",
"severity": "warning",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.38.0"
},
"annotations": {
"description": "Check publisher and PostgreSQL availability before trusting queue gauges.",
"summary": "Publisher backlog observations are stale"
},
"state": "firing",
"activeAt": "2026-11-20T17:38:01.478517494Z",
"value": "6.370885992050171e+01"
}
]
}