Record SLO and observability fault experiments
This commit is contained in:
@@ -0,0 +1,324 @@
|
||||
{
|
||||
"prometheus_samples": [
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "inactive",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:23:44+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 0.0,
|
||||
"ready_slow_ratio_30s": 0.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "inactive",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:23:49+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 0.0,
|
||||
"ready_slow_ratio_30s": 0.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "inactive",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:23:54+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 0.0,
|
||||
"ready_slow_ratio_30s": 0.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "inactive",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:23:59+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 0.0,
|
||||
"ready_slow_ratio_30s": 0.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "inactive",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:24:04+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 4.44206,
|
||||
"ready_slow_ratio_30s": 0.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "inactive",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:24:09+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 9.447189,
|
||||
"ready_slow_ratio_30s": 0.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "inactive",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:24:14+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 14.45216,
|
||||
"ready_slow_ratio_30s": 0.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "pending",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:24:19+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 19.454665,
|
||||
"ready_slow_ratio_30s": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "pending",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:24:24+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 24.459164,
|
||||
"ready_slow_ratio_30s": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "firing",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:24:29+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 29.464961,
|
||||
"ready_slow_ratio_30s": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "firing",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:24:34+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 34.474193,
|
||||
"ready_slow_ratio_30s": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "firing",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:24:39+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 0.0,
|
||||
"ready_slow_ratio_30s": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "inactive",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:24:44+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 0.0,
|
||||
"ready_slow_ratio_30s": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "inactive",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:24:49+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 0.0,
|
||||
"ready_slow_ratio_30s": 0.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "inactive",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:24:54+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 0.0,
|
||||
"ready_slow_ratio_30s": 0.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "inactive",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:24:59+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 0.0,
|
||||
"ready_slow_ratio_30s": 0.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "inactive",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:25:04+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 0.0,
|
||||
"ready_slow_ratio_30s": 0.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"alerts": {
|
||||
"Lab6ApiBurnLong": "inactive",
|
||||
"Lab6ApiBurnShort": "inactive",
|
||||
"Lab6HighHttpErrorRate": "inactive",
|
||||
"Lab6QueueLag": "inactive",
|
||||
"Lab6ReadyBurnLong": "inactive",
|
||||
"Lab6ReadyBurnShort": "inactive"
|
||||
},
|
||||
"timestamp": "2026-10-02T13:25:09+00:00",
|
||||
"values": {
|
||||
"collector_failed_spans": null,
|
||||
"http_5xx_ratio_30s": 0.0,
|
||||
"queue_oldest_seconds": 0.0,
|
||||
"ready_slow_ratio_30s": 0.0
|
||||
}
|
||||
}
|
||||
],
|
||||
"queue_alert_states": [
|
||||
"firing",
|
||||
"inactive",
|
||||
"pending"
|
||||
],
|
||||
"queued_uploads": 8,
|
||||
"ready_uploads": 8,
|
||||
"times": {
|
||||
"all_ready": "2026-10-02T13:24:33+00:00",
|
||||
"started": "2026-10-02T13:23:44+00:00",
|
||||
"uploads_accepted": "2026-10-02T13:23:54+00:00",
|
||||
"worker_started": "2026-10-02T13:24:32+00:00",
|
||||
"worker_stopped": "2026-10-02T13:23:54+00:00"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"accepted_ready": 40,
|
||||
"concurrency": 4,
|
||||
"elapsed_seconds": 29.794,
|
||||
"failed": 0,
|
||||
"failure_types": [],
|
||||
"lab": 6,
|
||||
"per_size": 20,
|
||||
"profiles": {
|
||||
"noise_800x600": {
|
||||
"ready_ms": {
|
||||
"p50": 4105.489,
|
||||
"p95": 4756.876,
|
||||
"p99": 5080.984
|
||||
},
|
||||
"size_bytes": 1441103,
|
||||
"successful": 20,
|
||||
"upload_ms": {
|
||||
"p50": 29.131,
|
||||
"p95": 52.165,
|
||||
"p99": 53.056
|
||||
}
|
||||
},
|
||||
"small_320x200": {
|
||||
"ready_ms": {
|
||||
"p50": 1573.785,
|
||||
"p95": 1916.581,
|
||||
"p99": 1919.551
|
||||
},
|
||||
"size_bytes": 111817,
|
||||
"successful": 20,
|
||||
"upload_ms": {
|
||||
"p50": 17.66,
|
||||
"p95": 27.092,
|
||||
"p99": 66.386
|
||||
}
|
||||
}
|
||||
},
|
||||
"workload": "synthetic PNG upload and poll until ready"
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"accepted_ready": 30,
|
||||
"concurrency": 4,
|
||||
"elapsed_seconds": 72.988,
|
||||
"failed": 0,
|
||||
"failure_types": [],
|
||||
"lab": 6,
|
||||
"per_size": 30,
|
||||
"profiles": {
|
||||
"noise_800x600": {
|
||||
"ready_ms": {
|
||||
"p50": 9476.253,
|
||||
"p95": 10359.763,
|
||||
"p99": 10437.545
|
||||
},
|
||||
"size_bytes": 1441103,
|
||||
"successful": 30,
|
||||
"upload_ms": {
|
||||
"p50": 27.409,
|
||||
"p95": 68.138,
|
||||
"p99": 83.338
|
||||
}
|
||||
}
|
||||
},
|
||||
"workload": "synthetic PNG upload and poll until ready"
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:31+00:00", "values": {"collector_failed_spans": 20872.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:36+00:00", "values": {"collector_failed_spans": 20878.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:41+00:00", "values": {"collector_failed_spans": 20900.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:46+00:00", "values": {"collector_failed_spans": 20912.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:51+00:00", "values": {"collector_failed_spans": 20918.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:56+00:00", "values": {"collector_failed_spans": 20930.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:36:01+00:00", "values": {"collector_failed_spans": 20930.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.699382, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:36:06+00:00", "values": {"collector_failed_spans": 21312.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.699382, "ready_slow_ratio_30s": 0.6666666666666666}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:36:11+00:00", "values": {"collector_failed_spans": 21914.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.303315, "ready_slow_ratio_30s": 0.8333333333333333}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "pending"}, "timestamp": "2026-10-02T13:36:16+00:00", "values": {"collector_failed_spans": 22138.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.303315, "ready_slow_ratio_30s": 0.875}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "pending"}, "timestamp": "2026-10-02T13:36:21+00:00", "values": {"collector_failed_spans": 22264.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.819518, "ready_slow_ratio_30s": 0.9}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:26+00:00", "values": {"collector_failed_spans": 22264.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.819518, "ready_slow_ratio_30s": 1.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:31+00:00", "values": {"collector_failed_spans": 22752.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.800324, "ready_slow_ratio_30s": 1.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:36+00:00", "values": {"collector_failed_spans": 22807.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.800324, "ready_slow_ratio_30s": 1.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:41+00:00", "values": {"collector_failed_spans": 22807.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 6.705292, "ready_slow_ratio_30s": 1.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:46+00:00", "values": {"collector_failed_spans": 22817.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 6.705292, "ready_slow_ratio_30s": 1.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:51+00:00", "values": {"collector_failed_spans": 22846.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.390955, "ready_slow_ratio_30s": 1.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:56+00:00", "values": {"collector_failed_spans": 22852.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.390955, "ready_slow_ratio_30s": 1.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:37:01+00:00", "values": {"collector_failed_spans": 22870.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.015189, "ready_slow_ratio_30s": 1.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:37:06+00:00", "values": {"collector_failed_spans": 22876.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.015189, "ready_slow_ratio_30s": 1.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:37:11+00:00", "values": {"collector_failed_spans": 22888.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:37:16+00:00", "values": {"collector_failed_spans": 22899.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:21+00:00", "values": {"collector_failed_spans": 22917.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:26+00:00", "values": {"collector_failed_spans": 22929.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:31+00:00", "values": {"collector_failed_spans": 22939.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:36+00:00", "values": {"collector_failed_spans": 22951.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:41+00:00", "values": {"collector_failed_spans": 22963.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:46+00:00", "values": {"collector_failed_spans": 22975.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:51+00:00", "values": {"collector_failed_spans": 22987.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:56+00:00", "values": {"collector_failed_spans": 22993.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:38:01+00:00", "values": {"collector_failed_spans": 23016.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:38:06+00:00", "values": {"collector_failed_spans": 23022.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
@@ -0,0 +1,30 @@
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:32:53+00:00", "values": {"collector_failed_spans": 20481.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:32:58+00:00", "values": {"collector_failed_spans": 20491.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:03+00:00", "values": {"collector_failed_spans": 20509.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:08+00:00", "values": {"collector_failed_spans": 20527.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:13+00:00", "values": {"collector_failed_spans": 20531.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:18+00:00", "values": {"collector_failed_spans": 20537.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:23+00:00", "values": {"collector_failed_spans": 20549.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:28+00:00", "values": {"collector_failed_spans": 20561.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:33+00:00", "values": {"collector_failed_spans": 20585.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.155266, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:38+00:00", "values": {"collector_failed_spans": 20591.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.998314, "ready_slow_ratio_30s": 0.05458974358974353}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:43+00:00", "values": {"collector_failed_spans": 20609.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 2.081209, "ready_slow_ratio_30s": 0.09646666666666656}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:48+00:00", "values": {"collector_failed_spans": 20627.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 2.683843, "ready_slow_ratio_30s": 0.1055743589743589}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:53+00:00", "values": {"collector_failed_spans": 20637.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 2.220981, "ready_slow_ratio_30s": 0.11358709677419333}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:58+00:00", "values": {"collector_failed_spans": 20643.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 2.589964, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:03+00:00", "values": {"collector_failed_spans": 20655.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:08+00:00", "values": {"collector_failed_spans": 20661.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:14+00:00", "values": {"collector_failed_spans": 20679.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:19+00:00", "values": {"collector_failed_spans": 20701.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:24+00:00", "values": {"collector_failed_spans": 20707.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:29+00:00", "values": {"collector_failed_spans": 20719.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:34+00:00", "values": {"collector_failed_spans": 20731.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:39+00:00", "values": {"collector_failed_spans": 20743.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:44+00:00", "values": {"collector_failed_spans": 20773.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:49+00:00", "values": {"collector_failed_spans": 20773.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:54+00:00", "values": {"collector_failed_spans": 20773.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:59+00:00", "values": {"collector_failed_spans": 20797.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:04+00:00", "values": {"collector_failed_spans": 20803.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:09+00:00", "values": {"collector_failed_spans": 20821.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:14+00:00", "values": {"collector_failed_spans": 20838.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:19+00:00", "values": {"collector_failed_spans": 20848.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,6 @@
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:28+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:33+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:38+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:43+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:48+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:53+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,150 @@
|
||||
{
|
||||
"collector_counter_delta": {
|
||||
"otelcol_exporter_enqueue_failed_spans": 19486.0,
|
||||
"otelcol_exporter_send_failed_log_records": 11063.0,
|
||||
"otelcol_exporter_send_failed_metric_points": 484.0,
|
||||
"otelcol_exporter_send_failed_spans": 16547.0,
|
||||
"otelcol_receiver_refused_log_records": 0.0,
|
||||
"otelcol_receiver_refused_metric_points": 0.0,
|
||||
"otelcol_receiver_refused_spans": 0.0
|
||||
},
|
||||
"collector_counters_after": {
|
||||
"otelcol_exporter_enqueue_failed_spans": 112097.0,
|
||||
"otelcol_exporter_send_failed_log_records": 31979.0,
|
||||
"otelcol_exporter_send_failed_metric_points": 1570.0,
|
||||
"otelcol_exporter_send_failed_spans": 44902.0,
|
||||
"otelcol_receiver_refused_log_records": 0.0,
|
||||
"otelcol_receiver_refused_metric_points": 0.0,
|
||||
"otelcol_receiver_refused_spans": 0.0
|
||||
},
|
||||
"collector_counters_before": {
|
||||
"otelcol_exporter_enqueue_failed_spans": 92611.0,
|
||||
"otelcol_exporter_send_failed_log_records": 20916.0,
|
||||
"otelcol_exporter_send_failed_metric_points": 1086.0,
|
||||
"otelcol_exporter_send_failed_spans": 28355.0,
|
||||
"otelcol_receiver_refused_log_records": 0.0,
|
||||
"otelcol_receiver_refused_metric_points": 0.0,
|
||||
"otelcol_receiver_refused_spans": 0.0
|
||||
},
|
||||
"load_passed": true,
|
||||
"smoke_passed": true,
|
||||
"smoke_tail": [
|
||||
"python3 tests/smoke.py --base-url \"http://127.0.0.1:8080\" ",
|
||||
"PASS: lab6 public API scenario; infrastructure and grading evidence remain separate."
|
||||
],
|
||||
"started": "2026-10-02T14:01:35+00:00",
|
||||
"stats_after_35s": [
|
||||
{
|
||||
"BlockIO": "2.23GB / 0B",
|
||||
"CPUPerc": "1.21%",
|
||||
"Container": "mtusi2026lab6-otel-collector-1",
|
||||
"ID": "3bedc51054f5",
|
||||
"MemPerc": "30.20%",
|
||||
"MemUsage": "116MiB / 384MiB",
|
||||
"Name": "mtusi2026lab6-otel-collector-1",
|
||||
"NetIO": "126MB / 37.1MB",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"BlockIO": "40.6MB / 0B",
|
||||
"CPUPerc": "14.25%",
|
||||
"Container": "mtusi2026lab6-api-1",
|
||||
"ID": "2ad72f3266ec",
|
||||
"MemPerc": "32.39%",
|
||||
"MemUsage": "82.91MiB / 256MiB",
|
||||
"Name": "mtusi2026lab6-api-1",
|
||||
"NetIO": "22.6MB / 29.5MB",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"BlockIO": "11.3MB / 0B",
|
||||
"CPUPerc": "0.09%",
|
||||
"Container": "mtusi2026lab6-worker-1",
|
||||
"ID": "88e67b14961c",
|
||||
"MemPerc": "7.12%",
|
||||
"MemUsage": "54.67MiB / 768MiB",
|
||||
"Name": "mtusi2026lab6-worker-1",
|
||||
"NetIO": "334kB / 453kB",
|
||||
"PIDs": "8"
|
||||
}
|
||||
],
|
||||
"stats_before": [
|
||||
{
|
||||
"BlockIO": "2.21GB / 0B",
|
||||
"CPUPerc": "0.19%",
|
||||
"Container": "mtusi2026lab6-otel-collector-1",
|
||||
"ID": "3bedc51054f5",
|
||||
"MemPerc": "18.44%",
|
||||
"MemUsage": "70.82MiB / 384MiB",
|
||||
"Name": "mtusi2026lab6-otel-collector-1",
|
||||
"NetIO": "110MB / 36.8MB",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"BlockIO": "35.6MB / 0B",
|
||||
"CPUPerc": "8.60%",
|
||||
"Container": "mtusi2026lab6-api-1",
|
||||
"ID": "2ad72f3266ec",
|
||||
"MemPerc": "22.86%",
|
||||
"MemUsage": "58.52MiB / 256MiB",
|
||||
"Name": "mtusi2026lab6-api-1",
|
||||
"NetIO": "1.13MB / 1.78MB",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"BlockIO": "10.8MB / 0B",
|
||||
"CPUPerc": "0.15%",
|
||||
"Container": "mtusi2026lab6-worker-1",
|
||||
"ID": "88e67b14961c",
|
||||
"MemPerc": "7.09%",
|
||||
"MemUsage": "54.44MiB / 768MiB",
|
||||
"Name": "mtusi2026lab6-worker-1",
|
||||
"NetIO": "188kB / 267kB",
|
||||
"PIDs": "8"
|
||||
}
|
||||
],
|
||||
"stats_during": [
|
||||
{
|
||||
"BlockIO": "2.23GB / 0B",
|
||||
"CPUPerc": "2.40%",
|
||||
"Container": "mtusi2026lab6-otel-collector-1",
|
||||
"ID": "3bedc51054f5",
|
||||
"MemPerc": "27.14%",
|
||||
"MemUsage": "104.2MiB / 384MiB",
|
||||
"Name": "mtusi2026lab6-otel-collector-1",
|
||||
"NetIO": "126MB / 37MB",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"BlockIO": "40.6MB / 0B",
|
||||
"CPUPerc": "7.68%",
|
||||
"Container": "mtusi2026lab6-api-1",
|
||||
"ID": "2ad72f3266ec",
|
||||
"MemPerc": "32.65%",
|
||||
"MemUsage": "83.58MiB / 256MiB",
|
||||
"Name": "mtusi2026lab6-api-1",
|
||||
"NetIO": "22.5MB / 29.3MB",
|
||||
"PIDs": "13"
|
||||
},
|
||||
{
|
||||
"BlockIO": "11.3MB / 0B",
|
||||
"CPUPerc": "0.14%",
|
||||
"Container": "mtusi2026lab6-worker-1",
|
||||
"ID": "88e67b14961c",
|
||||
"MemPerc": "7.12%",
|
||||
"MemUsage": "54.67MiB / 768MiB",
|
||||
"Name": "mtusi2026lab6-worker-1",
|
||||
"NetIO": "325kB / 431kB",
|
||||
"PIDs": "8"
|
||||
}
|
||||
],
|
||||
"stores": [
|
||||
"prometheus",
|
||||
"loki",
|
||||
"tempo"
|
||||
],
|
||||
"stores_healthy": true,
|
||||
"stores_restored": true,
|
||||
"stores_restored_at": "2026-10-02T14:03:01+00:00",
|
||||
"stores_stopped": "2026-10-02T14:01:48+00:00"
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.017,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 17876,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 595.531,
|
||||
"successful_latency_ms": {
|
||||
"p50": 4.844,
|
||||
"p95": 16.108,
|
||||
"p99": 49.174
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.011,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 18860,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 628.445,
|
||||
"successful_latency_ms": {
|
||||
"p50": 4.677,
|
||||
"p95": 14.469,
|
||||
"p99": 44.625
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -41,6 +41,7 @@ def main():
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--concurrency", type=int, default=4)
|
||||
parser.add_argument("--per-size", type=int, default=4)
|
||||
parser.add_argument("--profile", choices=("small_320x200", "noise_800x600"))
|
||||
parser.add_argument("--output", required=True)
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.concurrency <= 16 or not 1 <= args.per_size <= 100:
|
||||
@@ -52,6 +53,8 @@ def main():
|
||||
"small_320x200": (ROOT / "tests/fixtures/sample.png").read_bytes(),
|
||||
"noise_800x600": png_noise(800, 600),
|
||||
}
|
||||
if args.profile:
|
||||
images = {args.profile: images[args.profile]}
|
||||
items = [(name, data) for name, data in images.items() for _ in range(args.per_size)]
|
||||
|
||||
def work(item):
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove business API survives a bounded outage of all telemetry stores."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import urlopen
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FILES = ["-f", "compose.yaml"]
|
||||
if (ROOT / "compose.local.yaml").exists():
|
||||
FILES += ["-f", "compose.local.yaml"]
|
||||
FILES += ["-f", "compose.telemetry.yaml"]
|
||||
COMPOSE = ["docker", "compose", *FILES, "--env-file", ".env"]
|
||||
COUNTERS = (
|
||||
"otelcol_exporter_enqueue_failed_spans",
|
||||
"otelcol_exporter_send_failed_spans",
|
||||
"otelcol_exporter_send_failed_log_records",
|
||||
"otelcol_exporter_send_failed_metric_points",
|
||||
"otelcol_receiver_refused_spans",
|
||||
"otelcol_receiver_refused_log_records",
|
||||
"otelcol_receiver_refused_metric_points",
|
||||
)
|
||||
|
||||
|
||||
def run(*args):
|
||||
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True)
|
||||
|
||||
|
||||
def stats():
|
||||
result = run("docker", "stats", "--no-stream", "--format", "{{json .}}",
|
||||
"mtusi2026lab6-otel-collector-1", "mtusi2026lab6-api-1", "mtusi2026lab6-worker-1")
|
||||
return [json.loads(line) for line in result.stdout.splitlines()] if result.returncode == 0 else []
|
||||
|
||||
|
||||
def metric(name):
|
||||
url = "http://127.0.0.1:9090/api/v1/query?" + urlencode({"query": f"sum({name})"})
|
||||
try:
|
||||
with urlopen(url, timeout=5) as response:
|
||||
rows = json.load(response)["data"]["result"]
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
return float(rows[0]["value"][1]) if rows else None
|
||||
|
||||
|
||||
def stores_up():
|
||||
url = "http://127.0.0.1:9090/api/v1/query?" + urlencode({"query": "up{job=~\"prometheus|loki|tempo\"}"})
|
||||
try:
|
||||
with urlopen(url, timeout=5) as response:
|
||||
rows = json.load(response)["data"]["result"]
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
return len(rows) == 3 and all(row["value"][1] == "1" for row in rows)
|
||||
|
||||
|
||||
def main():
|
||||
output = ROOT / "evidence/telemetry-backend-outage-2026-10-02.json"
|
||||
data = {"started": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||
"stores": ["prometheus", "loki", "tempo"], "stats_before": stats(),
|
||||
"collector_counters_before": {name: metric(name) for name in COUNTERS}}
|
||||
stopped = False
|
||||
try:
|
||||
result = run(*COMPOSE, "stop", "prometheus", "loki", "tempo")
|
||||
if result.returncode:
|
||||
raise RuntimeError(result.stderr[-500:])
|
||||
stopped = True
|
||||
data["stores_stopped"] = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
smoke = run("make", "test", "BASE_URL=http://127.0.0.1:8080")
|
||||
data["smoke_passed"] = smoke.returncode == 0
|
||||
data["smoke_tail"] = (smoke.stdout + smoke.stderr).splitlines()[-5:]
|
||||
load = run("python3", "tests/load.py", "--base-url", "http://127.0.0.1:8080",
|
||||
"--duration", "30", "--concurrency", "4",
|
||||
"--output", "evidence/telemetry-outage-http-load-2026-10-02.json")
|
||||
data["load_passed"] = load.returncode == 0
|
||||
data["stats_during"] = stats()
|
||||
# Longer than Collector's 30-second exporter retry horizon.
|
||||
time.sleep(35)
|
||||
data["stats_after_35s"] = stats()
|
||||
finally:
|
||||
if stopped:
|
||||
restored = run(*COMPOSE, "up", "-d", "prometheus", "loki", "tempo")
|
||||
data["stores_restored"] = restored.returncode == 0
|
||||
data["stores_restored_at"] = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
healthy_since = None
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
if stores_up():
|
||||
healthy_since = healthy_since or time.monotonic()
|
||||
if time.monotonic() - healthy_since >= 30:
|
||||
break
|
||||
else:
|
||||
healthy_since = None
|
||||
time.sleep(5)
|
||||
data["stores_healthy"] = (healthy_since is not None and
|
||||
time.monotonic() - healthy_since >= 30 and stores_up())
|
||||
data["collector_counters_after"] = {name: metric(name) for name in COUNTERS}
|
||||
data["collector_counter_delta"] = {
|
||||
name: (data["collector_counters_after"][name] - data["collector_counters_before"][name]
|
||||
if data["collector_counters_before"][name] is not None and
|
||||
data["collector_counters_after"][name] is not None else None)
|
||||
for name in COUNTERS
|
||||
}
|
||||
output.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"smoke_passed": data.get("smoke_passed"),
|
||||
"load_passed": data.get("load_passed"),
|
||||
"stores_restored": data.get("stores_restored"),
|
||||
"collector_counter_delta": data["collector_counter_delta"],
|
||||
"output": str(output)}, sort_keys=True))
|
||||
if not all(data.get(key) for key in ("smoke_passed", "load_passed", "stores_restored", "stores_healthy")):
|
||||
raise SystemExit("business path or telemetry recovery failed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Stop worker, build a real durable backlog, then verify queue-lag recovery."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
from client import Client # noqa: E402
|
||||
from slo_watch import snapshot # noqa: E402
|
||||
|
||||
|
||||
FILES = ["-f", "compose.yaml"]
|
||||
if (ROOT / "compose.local.yaml").exists():
|
||||
FILES += ["-f", "compose.local.yaml"]
|
||||
FILES += ["-f", "compose.telemetry.yaml"]
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(["docker", "compose", *FILES, "--env-file", ".env", *args],
|
||||
cwd=ROOT, check=True, stdout=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def now():
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def main():
|
||||
client = Client("http://127.0.0.1:8080")
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
image = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
samples = []
|
||||
stop = threading.Event()
|
||||
|
||||
def watch():
|
||||
while not stop.is_set():
|
||||
samples.append(snapshot("http://127.0.0.1:9090"))
|
||||
stop.wait(5)
|
||||
|
||||
watcher = threading.Thread(target=watch, daemon=True)
|
||||
watcher.start()
|
||||
times = {"started": now()}
|
||||
worker_stopped = False
|
||||
try:
|
||||
compose("stop", "worker")
|
||||
worker_stopped = True
|
||||
times["worker_stopped"] = now()
|
||||
attachments = []
|
||||
for _ in range(8):
|
||||
response = client.call(
|
||||
"POST", f"/api/v1/chats/{chat['id']}/attachments?filename=queue-probe.png",
|
||||
202, actor=alice, raw=image, headers={"Content-Type": "image/png"}, schema="Attachment",
|
||||
)
|
||||
attachments.append(response["json"])
|
||||
times["uploads_accepted"] = now()
|
||||
time.sleep(35)
|
||||
compose("up", "-d", "worker")
|
||||
worker_stopped = False
|
||||
times["worker_started"] = now()
|
||||
for item in attachments:
|
||||
client.wait_ready(alice, item, wait_seconds=120)
|
||||
times["all_ready"] = now()
|
||||
time.sleep(40)
|
||||
finally:
|
||||
if worker_stopped:
|
||||
compose("up", "-d", "worker")
|
||||
stop.set()
|
||||
watcher.join(timeout=6)
|
||||
states = [item["alerts"].get("Lab6QueueLag") for item in samples]
|
||||
data = {"times": times, "queued_uploads": len(attachments),
|
||||
"ready_uploads": len(attachments), "prometheus_samples": samples,
|
||||
"queue_alert_states": sorted(set(states))}
|
||||
path = ROOT / "evidence/queue-worker-stop-2026-10-02.json"
|
||||
path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"queued_uploads": len(attachments), "states": data["queue_alert_states"],
|
||||
"times": times, "output": str(path)}, sort_keys=True))
|
||||
if "firing" not in states or states[-1] != "inactive":
|
||||
raise SystemExit("queue lag alert did not fire and resolve")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reproduce a DB outage and capture Prometheus burn-rate transitions."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
COMPOSE_FILES = ["-f", "compose.yaml"]
|
||||
if (ROOT / "compose.local.yaml").exists():
|
||||
COMPOSE_FILES += ["-f", "compose.local.yaml"]
|
||||
COMPOSE_FILES += ["-f", "compose.telemetry.yaml"]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
from slo_watch import snapshot # noqa: E402
|
||||
|
||||
|
||||
PROBE = r'''
|
||||
import base64, json, sys, time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
end = time.monotonic() + int(sys.argv[1])
|
||||
events = []
|
||||
headers = {"Authorization": "Basic " + base64.b64encode(b"lab6_probe:synthetic-invalid-password").decode()}
|
||||
def run_one(_):
|
||||
result = []
|
||||
while time.monotonic() < end:
|
||||
req = Request("http://127.0.0.1:8080/api/v1/auth/sessions", data=b"", headers=headers, method="POST")
|
||||
try:
|
||||
with urlopen(req, timeout=30) as response:
|
||||
code = response.status
|
||||
except HTTPError as exc:
|
||||
code = exc.code
|
||||
except OSError:
|
||||
code = 0
|
||||
result.append([datetime.now(timezone.utc).isoformat(timespec="seconds"), code])
|
||||
time.sleep(0.4)
|
||||
return result
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
for batch in pool.map(run_one, range(1)):
|
||||
events.extend(batch)
|
||||
print(json.dumps({"events": sorted(events)}))
|
||||
'''
|
||||
|
||||
|
||||
def utc_now():
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(
|
||||
["docker", "compose", *COMPOSE_FILES, "--env-file", ".env", *args],
|
||||
cwd=ROOT, check=True, stdout=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--case", choices=("short", "long"), required=True)
|
||||
parser.add_argument("--output", required=True)
|
||||
args = parser.parse_args()
|
||||
pre, outage, post = (15, 50, 90) if args.case == "short" else (15, 180, 130)
|
||||
duration = pre + outage + post + 10
|
||||
samples, watch_errors, times = [], [], {"started": utc_now()}
|
||||
stop_watching = threading.Event()
|
||||
|
||||
def watch():
|
||||
while not stop_watching.is_set():
|
||||
try:
|
||||
samples.append(snapshot("http://127.0.0.1:9090"))
|
||||
except (OSError, ValueError, KeyError) as exc:
|
||||
watch_errors.append({"timestamp": utc_now(), "type": type(exc).__name__})
|
||||
stop_watching.wait(5)
|
||||
|
||||
watcher = threading.Thread(target=watch, daemon=True)
|
||||
watcher.start()
|
||||
command = [
|
||||
"docker", "compose", *COMPOSE_FILES, "--env-file", ".env", "exec", "-T",
|
||||
"api", "python", "-u", "-", str(duration),
|
||||
]
|
||||
probe = subprocess.Popen(command, cwd=ROOT, stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
probe.stdin.write(PROBE)
|
||||
probe.stdin.close()
|
||||
db_stopped = False
|
||||
try:
|
||||
time.sleep(pre)
|
||||
times["stop_requested"] = utc_now()
|
||||
compose("stop", "postgres")
|
||||
db_stopped = True
|
||||
times["db_stopped"] = utc_now()
|
||||
time.sleep(outage)
|
||||
times["restart_requested"] = utc_now()
|
||||
compose("up", "-d", "--wait", "postgres")
|
||||
db_stopped = False
|
||||
times["db_healthy"] = utc_now()
|
||||
time.sleep(post)
|
||||
finally:
|
||||
if db_stopped:
|
||||
compose("up", "-d", "--wait", "postgres")
|
||||
times["db_healthy_after_cleanup"] = utc_now()
|
||||
stop_watching.set()
|
||||
watcher.join(timeout=6)
|
||||
|
||||
output = probe.stdout.read()
|
||||
error = probe.stderr.read()
|
||||
probe.wait(timeout=15)
|
||||
result = json.loads(output) if probe.returncode == 0 else {"error": error[-500:]}
|
||||
result.update({"case": args.case, "planned_seconds": {"pre": pre, "outage": outage, "post": post},
|
||||
"times": times, "prometheus_samples": samples, "watch_errors": watch_errors})
|
||||
path = Path(args.output)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
states = {name: {sample["alerts"].get(name) for sample in samples}
|
||||
for name in ("Lab6HighHttpErrorRate", "Lab6ApiBurnShort", "Lab6ApiBurnLong")}
|
||||
codes = {event[1] for event in result.get("events", [])}
|
||||
print(json.dumps({"case": args.case, "codes": sorted(codes),
|
||||
"states": {name: sorted(values) for name, values in states.items()},
|
||||
"samples": len(samples), "output": str(path)}, sort_keys=True))
|
||||
expected = {"Lab6HighHttpErrorRate", "Lab6ApiBurnShort"}
|
||||
if args.case == "long":
|
||||
expected.add("Lab6ApiBurnLong")
|
||||
if probe.returncode or not {401, 503} <= codes or any("firing" not in states[name] for name in expected):
|
||||
raise SystemExit("incident did not demonstrate the expected HTTP SLO transitions")
|
||||
if any(sample["alerts"].get(name) != "inactive" for name in expected for sample in samples[-3:]):
|
||||
raise SystemExit("alerts did not resolve after database recovery")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user