From f61d0705749a7ca385a6eba4b28fc2c20a0cb925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 17:06:45 +0300 Subject: [PATCH] Record SLO and observability fault experiments --- evidence/queue-worker-stop-2026-10-02.json | 324 +++ evidence/ready-throttle-load-2026-10-02.json | 40 + ...ready-throttle-strong-load-2026-10-02.json | 26 + ...ady-throttle-strong-watch-2026-10-02.jsonl | 32 + .../ready-throttle-watch-2026-10-02.jsonl | 30 + evidence/slo-long-db-2026-10-02.json | 2262 +++++++++++++++++ evidence/slo-quiet-baseline-2026-10-02.jsonl | 6 + evidence/slo-short-db-2026-10-02.json | 1584 ++++++++++++ .../telemetry-backend-outage-2026-10-02.json | 150 ++ ...telemetry-outage-http-load-2026-10-02.json | 17 + evidence/tempo-recovery-load-2026-10-02.json | 17 + scripts/measure-image-pipeline.py | 3 + student-tests/test_observability_failure.py | 118 + student-tests/test_queue_incident.py | 89 + student-tests/test_slo_alerts.py | 138 + 15 files changed, 4836 insertions(+) create mode 100644 evidence/queue-worker-stop-2026-10-02.json create mode 100644 evidence/ready-throttle-load-2026-10-02.json create mode 100644 evidence/ready-throttle-strong-load-2026-10-02.json create mode 100644 evidence/ready-throttle-strong-watch-2026-10-02.jsonl create mode 100644 evidence/ready-throttle-watch-2026-10-02.jsonl create mode 100644 evidence/slo-long-db-2026-10-02.json create mode 100644 evidence/slo-quiet-baseline-2026-10-02.jsonl create mode 100644 evidence/slo-short-db-2026-10-02.json create mode 100644 evidence/telemetry-backend-outage-2026-10-02.json create mode 100644 evidence/telemetry-outage-http-load-2026-10-02.json create mode 100644 evidence/tempo-recovery-load-2026-10-02.json create mode 100644 student-tests/test_observability_failure.py create mode 100644 student-tests/test_queue_incident.py create mode 100644 student-tests/test_slo_alerts.py diff --git a/evidence/queue-worker-stop-2026-10-02.json b/evidence/queue-worker-stop-2026-10-02.json new file mode 100644 index 0000000..0abbac5 --- /dev/null +++ b/evidence/queue-worker-stop-2026-10-02.json @@ -0,0 +1,324 @@ +{ + "prometheus_samples": [ + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:23:44+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:23:49+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:23:54+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:23:59+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:04+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 4.44206, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:09+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 9.447189, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:14+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 14.45216, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "pending", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:19+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 19.454665, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "pending", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:24+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 24.459164, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "firing", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:29+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 29.464961, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "firing", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:34+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 34.474193, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "firing", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:39+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:44+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:49+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:54+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:59+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:25:04+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:25:09+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + } + ], + "queue_alert_states": [ + "firing", + "inactive", + "pending" + ], + "queued_uploads": 8, + "ready_uploads": 8, + "times": { + "all_ready": "2026-10-02T13:24:33+00:00", + "started": "2026-10-02T13:23:44+00:00", + "uploads_accepted": "2026-10-02T13:23:54+00:00", + "worker_started": "2026-10-02T13:24:32+00:00", + "worker_stopped": "2026-10-02T13:23:54+00:00" + } +} diff --git a/evidence/ready-throttle-load-2026-10-02.json b/evidence/ready-throttle-load-2026-10-02.json new file mode 100644 index 0000000..c63deac --- /dev/null +++ b/evidence/ready-throttle-load-2026-10-02.json @@ -0,0 +1,40 @@ +{ + "accepted_ready": 40, + "concurrency": 4, + "elapsed_seconds": 29.794, + "failed": 0, + "failure_types": [], + "lab": 6, + "per_size": 20, + "profiles": { + "noise_800x600": { + "ready_ms": { + "p50": 4105.489, + "p95": 4756.876, + "p99": 5080.984 + }, + "size_bytes": 1441103, + "successful": 20, + "upload_ms": { + "p50": 29.131, + "p95": 52.165, + "p99": 53.056 + } + }, + "small_320x200": { + "ready_ms": { + "p50": 1573.785, + "p95": 1916.581, + "p99": 1919.551 + }, + "size_bytes": 111817, + "successful": 20, + "upload_ms": { + "p50": 17.66, + "p95": 27.092, + "p99": 66.386 + } + } + }, + "workload": "synthetic PNG upload and poll until ready" +} diff --git a/evidence/ready-throttle-strong-load-2026-10-02.json b/evidence/ready-throttle-strong-load-2026-10-02.json new file mode 100644 index 0000000..6cba4bd --- /dev/null +++ b/evidence/ready-throttle-strong-load-2026-10-02.json @@ -0,0 +1,26 @@ +{ + "accepted_ready": 30, + "concurrency": 4, + "elapsed_seconds": 72.988, + "failed": 0, + "failure_types": [], + "lab": 6, + "per_size": 30, + "profiles": { + "noise_800x600": { + "ready_ms": { + "p50": 9476.253, + "p95": 10359.763, + "p99": 10437.545 + }, + "size_bytes": 1441103, + "successful": 30, + "upload_ms": { + "p50": 27.409, + "p95": 68.138, + "p99": 83.338 + } + } + }, + "workload": "synthetic PNG upload and poll until ready" +} diff --git a/evidence/ready-throttle-strong-watch-2026-10-02.jsonl b/evidence/ready-throttle-strong-watch-2026-10-02.jsonl new file mode 100644 index 0000000..23f8fa9 --- /dev/null +++ b/evidence/ready-throttle-strong-watch-2026-10-02.jsonl @@ -0,0 +1,32 @@ +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:31+00:00", "values": {"collector_failed_spans": 20872.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:36+00:00", "values": {"collector_failed_spans": 20878.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:41+00:00", "values": {"collector_failed_spans": 20900.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:46+00:00", "values": {"collector_failed_spans": 20912.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:51+00:00", "values": {"collector_failed_spans": 20918.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:56+00:00", "values": {"collector_failed_spans": 20930.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:36:01+00:00", "values": {"collector_failed_spans": 20930.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.699382, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:36:06+00:00", "values": {"collector_failed_spans": 21312.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.699382, "ready_slow_ratio_30s": 0.6666666666666666}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:36:11+00:00", "values": {"collector_failed_spans": 21914.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.303315, "ready_slow_ratio_30s": 0.8333333333333333}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "pending"}, "timestamp": "2026-10-02T13:36:16+00:00", "values": {"collector_failed_spans": 22138.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.303315, "ready_slow_ratio_30s": 0.875}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "pending"}, "timestamp": "2026-10-02T13:36:21+00:00", "values": {"collector_failed_spans": 22264.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.819518, "ready_slow_ratio_30s": 0.9}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:26+00:00", "values": {"collector_failed_spans": 22264.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.819518, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:31+00:00", "values": {"collector_failed_spans": 22752.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.800324, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:36+00:00", "values": {"collector_failed_spans": 22807.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.800324, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:41+00:00", "values": {"collector_failed_spans": 22807.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 6.705292, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:46+00:00", "values": {"collector_failed_spans": 22817.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 6.705292, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:51+00:00", "values": {"collector_failed_spans": 22846.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.390955, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:56+00:00", "values": {"collector_failed_spans": 22852.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.390955, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:37:01+00:00", "values": {"collector_failed_spans": 22870.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.015189, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:37:06+00:00", "values": {"collector_failed_spans": 22876.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.015189, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:37:11+00:00", "values": {"collector_failed_spans": 22888.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:37:16+00:00", "values": {"collector_failed_spans": 22899.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:21+00:00", "values": {"collector_failed_spans": 22917.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:26+00:00", "values": {"collector_failed_spans": 22929.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:31+00:00", "values": {"collector_failed_spans": 22939.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:36+00:00", "values": {"collector_failed_spans": 22951.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:41+00:00", "values": {"collector_failed_spans": 22963.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:46+00:00", "values": {"collector_failed_spans": 22975.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:51+00:00", "values": {"collector_failed_spans": 22987.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:56+00:00", "values": {"collector_failed_spans": 22993.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:38:01+00:00", "values": {"collector_failed_spans": 23016.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:38:06+00:00", "values": {"collector_failed_spans": 23022.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} diff --git a/evidence/ready-throttle-watch-2026-10-02.jsonl b/evidence/ready-throttle-watch-2026-10-02.jsonl new file mode 100644 index 0000000..9ac20cb --- /dev/null +++ b/evidence/ready-throttle-watch-2026-10-02.jsonl @@ -0,0 +1,30 @@ +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:32:53+00:00", "values": {"collector_failed_spans": 20481.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:32:58+00:00", "values": {"collector_failed_spans": 20491.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:03+00:00", "values": {"collector_failed_spans": 20509.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:08+00:00", "values": {"collector_failed_spans": 20527.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:13+00:00", "values": {"collector_failed_spans": 20531.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:18+00:00", "values": {"collector_failed_spans": 20537.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:23+00:00", "values": {"collector_failed_spans": 20549.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:28+00:00", "values": {"collector_failed_spans": 20561.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:33+00:00", "values": {"collector_failed_spans": 20585.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.155266, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:38+00:00", "values": {"collector_failed_spans": 20591.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.998314, "ready_slow_ratio_30s": 0.05458974358974353}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:43+00:00", "values": {"collector_failed_spans": 20609.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 2.081209, "ready_slow_ratio_30s": 0.09646666666666656}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:48+00:00", "values": {"collector_failed_spans": 20627.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 2.683843, "ready_slow_ratio_30s": 0.1055743589743589}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:53+00:00", "values": {"collector_failed_spans": 20637.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 2.220981, "ready_slow_ratio_30s": 0.11358709677419333}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:58+00:00", "values": {"collector_failed_spans": 20643.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 2.589964, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:03+00:00", "values": {"collector_failed_spans": 20655.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:08+00:00", "values": {"collector_failed_spans": 20661.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:14+00:00", "values": {"collector_failed_spans": 20679.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:19+00:00", "values": {"collector_failed_spans": 20701.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:24+00:00", "values": {"collector_failed_spans": 20707.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:29+00:00", "values": {"collector_failed_spans": 20719.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:34+00:00", "values": {"collector_failed_spans": 20731.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:39+00:00", "values": {"collector_failed_spans": 20743.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:44+00:00", "values": {"collector_failed_spans": 20773.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:49+00:00", "values": {"collector_failed_spans": 20773.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:54+00:00", "values": {"collector_failed_spans": 20773.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:59+00:00", "values": {"collector_failed_spans": 20797.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:04+00:00", "values": {"collector_failed_spans": 20803.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:09+00:00", "values": {"collector_failed_spans": 20821.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:14+00:00", "values": {"collector_failed_spans": 20838.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:19+00:00", "values": {"collector_failed_spans": 20848.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} diff --git a/evidence/slo-long-db-2026-10-02.json b/evidence/slo-long-db-2026-10-02.json new file mode 100644 index 0000000..cc27859 --- /dev/null +++ b/evidence/slo-long-db-2026-10-02.json @@ -0,0 +1,2262 @@ +{ + "case": "long", + "events": [ + [ + "2026-10-02T13:16:15+00:00", + 401 + ], + [ + "2026-10-02T13:16:15+00:00", + 401 + ], + [ + "2026-10-02T13:16:16+00:00", + 401 + ], + [ + "2026-10-02T13:16:16+00:00", + 401 + ], + [ + "2026-10-02T13:16:16+00:00", + 401 + ], + [ + "2026-10-02T13:16:17+00:00", + 401 + ], + [ + "2026-10-02T13:16:17+00:00", + 401 + ], + [ + "2026-10-02T13:16:18+00:00", + 401 + ], + [ + "2026-10-02T13:16:18+00:00", + 401 + ], + [ + "2026-10-02T13:16:18+00:00", + 401 + ], + [ + "2026-10-02T13:16:19+00:00", + 401 + ], + [ + "2026-10-02T13:16:19+00:00", + 401 + ], + [ + "2026-10-02T13:16:20+00:00", + 401 + ], + [ + "2026-10-02T13:16:20+00:00", + 401 + ], + [ + "2026-10-02T13:16:21+00:00", + 401 + ], + [ + "2026-10-02T13:16:21+00:00", + 401 + ], + [ + "2026-10-02T13:16:21+00:00", + 401 + ], + [ + "2026-10-02T13:16:22+00:00", + 401 + ], + [ + "2026-10-02T13:16:22+00:00", + 401 + ], + [ + "2026-10-02T13:16:23+00:00", + 401 + ], + [ + "2026-10-02T13:16:23+00:00", + 401 + ], + [ + "2026-10-02T13:16:23+00:00", + 401 + ], + [ + "2026-10-02T13:16:24+00:00", + 401 + ], + [ + "2026-10-02T13:16:24+00:00", + 401 + ], + [ + "2026-10-02T13:16:25+00:00", + 401 + ], + [ + "2026-10-02T13:16:25+00:00", + 401 + ], + [ + "2026-10-02T13:16:25+00:00", + 401 + ], + [ + "2026-10-02T13:16:26+00:00", + 401 + ], + [ + "2026-10-02T13:16:26+00:00", + 401 + ], + [ + "2026-10-02T13:16:27+00:00", + 401 + ], + [ + "2026-10-02T13:16:27+00:00", + 401 + ], + [ + "2026-10-02T13:16:27+00:00", + 401 + ], + [ + "2026-10-02T13:16:28+00:00", + 401 + ], + [ + "2026-10-02T13:16:28+00:00", + 401 + ], + [ + "2026-10-02T13:16:29+00:00", + 401 + ], + [ + "2026-10-02T13:16:29+00:00", + 401 + ], + [ + "2026-10-02T13:16:30+00:00", + 503 + ], + [ + "2026-10-02T13:16:30+00:00", + 503 + ], + [ + "2026-10-02T13:16:35+00:00", + 503 + ], + [ + "2026-10-02T13:16:41+00:00", + 503 + ], + [ + "2026-10-02T13:16:46+00:00", + 503 + ], + [ + "2026-10-02T13:16:52+00:00", + 503 + ], + [ + "2026-10-02T13:16:57+00:00", + 503 + ], + [ + "2026-10-02T13:17:02+00:00", + 503 + ], + [ + "2026-10-02T13:17:08+00:00", + 503 + ], + [ + "2026-10-02T13:17:13+00:00", + 503 + ], + [ + "2026-10-02T13:17:19+00:00", + 503 + ], + [ + "2026-10-02T13:17:24+00:00", + 503 + ], + [ + "2026-10-02T13:17:29+00:00", + 503 + ], + [ + "2026-10-02T13:17:35+00:00", + 503 + ], + [ + "2026-10-02T13:17:40+00:00", + 503 + ], + [ + "2026-10-02T13:17:46+00:00", + 503 + ], + [ + "2026-10-02T13:17:51+00:00", + 503 + ], + [ + "2026-10-02T13:17:57+00:00", + 503 + ], + [ + "2026-10-02T13:18:02+00:00", + 503 + ], + [ + "2026-10-02T13:18:07+00:00", + 503 + ], + [ + "2026-10-02T13:18:13+00:00", + 503 + ], + [ + "2026-10-02T13:18:18+00:00", + 503 + ], + [ + "2026-10-02T13:18:24+00:00", + 503 + ], + [ + "2026-10-02T13:18:29+00:00", + 503 + ], + [ + "2026-10-02T13:18:34+00:00", + 503 + ], + [ + "2026-10-02T13:18:40+00:00", + 503 + ], + [ + "2026-10-02T13:18:45+00:00", + 503 + ], + [ + "2026-10-02T13:18:51+00:00", + 503 + ], + [ + "2026-10-02T13:18:56+00:00", + 503 + ], + [ + "2026-10-02T13:19:01+00:00", + 503 + ], + [ + "2026-10-02T13:19:07+00:00", + 503 + ], + [ + "2026-10-02T13:19:12+00:00", + 503 + ], + [ + "2026-10-02T13:19:18+00:00", + 503 + ], + [ + "2026-10-02T13:19:23+00:00", + 503 + ], + [ + "2026-10-02T13:19:28+00:00", + 503 + ], + [ + "2026-10-02T13:19:34+00:00", + 503 + ], + [ + "2026-10-02T13:19:39+00:00", + 503 + ], + [ + "2026-10-02T13:19:45+00:00", + 503 + ], + [ + "2026-10-02T13:19:50+00:00", + 503 + ], + [ + "2026-10-02T13:19:55+00:00", + 503 + ], + [ + "2026-10-02T13:20:01+00:00", + 503 + ], + [ + "2026-10-02T13:20:06+00:00", + 503 + ], + [ + "2026-10-02T13:20:12+00:00", + 503 + ], + [ + "2026-10-02T13:20:17+00:00", + 503 + ], + [ + "2026-10-02T13:20:23+00:00", + 503 + ], + [ + "2026-10-02T13:20:28+00:00", + 503 + ], + [ + "2026-10-02T13:20:29+00:00", + 401 + ], + [ + "2026-10-02T13:20:30+00:00", + 401 + ], + [ + "2026-10-02T13:20:30+00:00", + 401 + ], + [ + "2026-10-02T13:20:30+00:00", + 401 + ], + [ + "2026-10-02T13:20:31+00:00", + 401 + ], + [ + "2026-10-02T13:20:31+00:00", + 401 + ], + [ + "2026-10-02T13:20:32+00:00", + 401 + ], + [ + "2026-10-02T13:20:32+00:00", + 401 + ], + [ + "2026-10-02T13:20:33+00:00", + 401 + ], + [ + "2026-10-02T13:20:33+00:00", + 401 + ], + [ + "2026-10-02T13:20:33+00:00", + 401 + ], + [ + "2026-10-02T13:20:34+00:00", + 401 + ], + [ + "2026-10-02T13:20:34+00:00", + 401 + ], + [ + "2026-10-02T13:20:35+00:00", + 401 + ], + [ + "2026-10-02T13:20:35+00:00", + 401 + ], + [ + "2026-10-02T13:20:35+00:00", + 401 + ], + [ + "2026-10-02T13:20:36+00:00", + 401 + ], + [ + "2026-10-02T13:20:36+00:00", + 401 + ], + [ + "2026-10-02T13:20:37+00:00", + 401 + ], + [ + "2026-10-02T13:20:37+00:00", + 401 + ], + [ + "2026-10-02T13:20:37+00:00", + 401 + ], + [ + "2026-10-02T13:20:38+00:00", + 401 + ], + [ + "2026-10-02T13:20:38+00:00", + 401 + ], + [ + "2026-10-02T13:20:39+00:00", + 401 + ], + [ + "2026-10-02T13:20:39+00:00", + 401 + ], + [ + "2026-10-02T13:20:39+00:00", + 401 + ], + [ + "2026-10-02T13:20:40+00:00", + 401 + ], + [ + "2026-10-02T13:20:40+00:00", + 401 + ], + [ + "2026-10-02T13:20:41+00:00", + 401 + ], + [ + "2026-10-02T13:20:41+00:00", + 401 + ], + [ + "2026-10-02T13:20:42+00:00", + 401 + ], + [ + "2026-10-02T13:20:42+00:00", + 401 + ], + [ + "2026-10-02T13:20:42+00:00", + 401 + ], + [ + "2026-10-02T13:20:43+00:00", + 401 + ], + [ + "2026-10-02T13:20:43+00:00", + 401 + ], + [ + "2026-10-02T13:20:44+00:00", + 401 + ], + [ + "2026-10-02T13:20:44+00:00", + 401 + ], + [ + "2026-10-02T13:20:44+00:00", + 401 + ], + [ + "2026-10-02T13:20:45+00:00", + 401 + ], + [ + "2026-10-02T13:20:45+00:00", + 401 + ], + [ + "2026-10-02T13:20:46+00:00", + 401 + ], + [ + "2026-10-02T13:20:46+00:00", + 401 + ], + [ + "2026-10-02T13:20:46+00:00", + 401 + ], + [ + "2026-10-02T13:20:47+00:00", + 401 + ], + [ + "2026-10-02T13:20:47+00:00", + 401 + ], + [ + "2026-10-02T13:20:48+00:00", + 401 + ], + [ + "2026-10-02T13:20:48+00:00", + 401 + ], + [ + "2026-10-02T13:20:48+00:00", + 401 + ], + [ + "2026-10-02T13:20:49+00:00", + 401 + ], + [ + "2026-10-02T13:20:49+00:00", + 401 + ], + [ + "2026-10-02T13:20:50+00:00", + 401 + ], + [ + "2026-10-02T13:20:50+00:00", + 401 + ], + [ + "2026-10-02T13:20:50+00:00", + 401 + ], + [ + "2026-10-02T13:20:51+00:00", + 401 + ], + [ + "2026-10-02T13:20:51+00:00", + 401 + ], + [ + "2026-10-02T13:20:52+00:00", + 401 + ], + [ + "2026-10-02T13:20:52+00:00", + 401 + ], + [ + "2026-10-02T13:20:53+00:00", + 401 + ], + [ + "2026-10-02T13:20:53+00:00", + 401 + ], + [ + "2026-10-02T13:20:53+00:00", + 401 + ], + [ + "2026-10-02T13:20:54+00:00", + 401 + ], + [ + "2026-10-02T13:20:54+00:00", + 401 + ], + [ + "2026-10-02T13:20:55+00:00", + 401 + ], + [ + "2026-10-02T13:20:55+00:00", + 401 + ], + [ + "2026-10-02T13:20:55+00:00", + 401 + ], + [ + "2026-10-02T13:20:56+00:00", + 401 + ], + [ + "2026-10-02T13:20:56+00:00", + 401 + ], + [ + "2026-10-02T13:20:57+00:00", + 401 + ], + [ + "2026-10-02T13:20:57+00:00", + 401 + ], + [ + "2026-10-02T13:20:57+00:00", + 401 + ], + [ + "2026-10-02T13:20:58+00:00", + 401 + ], + [ + "2026-10-02T13:20:58+00:00", + 401 + ], + [ + "2026-10-02T13:20:59+00:00", + 401 + ], + [ + "2026-10-02T13:20:59+00:00", + 401 + ], + [ + "2026-10-02T13:20:59+00:00", + 401 + ], + [ + "2026-10-02T13:21:00+00:00", + 401 + ], + [ + "2026-10-02T13:21:00+00:00", + 401 + ], + [ + "2026-10-02T13:21:01+00:00", + 401 + ], + [ + "2026-10-02T13:21:01+00:00", + 401 + ], + [ + "2026-10-02T13:21:01+00:00", + 401 + ], + [ + "2026-10-02T13:21:02+00:00", + 401 + ], + [ + "2026-10-02T13:21:02+00:00", + 401 + ], + [ + "2026-10-02T13:21:03+00:00", + 401 + ], + [ + "2026-10-02T13:21:03+00:00", + 401 + ], + [ + "2026-10-02T13:21:04+00:00", + 401 + ], + [ + "2026-10-02T13:21:04+00:00", + 401 + ], + [ + "2026-10-02T13:21:04+00:00", + 401 + ], + [ + "2026-10-02T13:21:05+00:00", + 401 + ], + [ + "2026-10-02T13:21:05+00:00", + 401 + ], + [ + "2026-10-02T13:21:06+00:00", + 401 + ], + [ + "2026-10-02T13:21:06+00:00", + 401 + ], + [ + "2026-10-02T13:21:06+00:00", + 401 + ], + [ + "2026-10-02T13:21:07+00:00", + 401 + ], + [ + "2026-10-02T13:21:07+00:00", + 401 + ], + [ + "2026-10-02T13:21:08+00:00", + 401 + ], + [ + "2026-10-02T13:21:08+00:00", + 401 + ], + [ + "2026-10-02T13:21:08+00:00", + 401 + ], + [ + "2026-10-02T13:21:09+00:00", + 401 + ], + [ + "2026-10-02T13:21:09+00:00", + 401 + ], + [ + "2026-10-02T13:21:10+00:00", + 401 + ], + [ + "2026-10-02T13:21:10+00:00", + 401 + ], + [ + "2026-10-02T13:21:10+00:00", + 401 + ], + [ + "2026-10-02T13:21:11+00:00", + 401 + ], + [ + "2026-10-02T13:21:11+00:00", + 401 + ], + [ + "2026-10-02T13:21:12+00:00", + 401 + ], + [ + "2026-10-02T13:21:12+00:00", + 401 + ], + [ + "2026-10-02T13:21:12+00:00", + 401 + ], + [ + "2026-10-02T13:21:13+00:00", + 401 + ], + [ + "2026-10-02T13:21:13+00:00", + 401 + ], + [ + "2026-10-02T13:21:14+00:00", + 401 + ], + [ + "2026-10-02T13:21:14+00:00", + 401 + ], + [ + "2026-10-02T13:21:15+00:00", + 401 + ], + [ + "2026-10-02T13:21:15+00:00", + 401 + ], + [ + "2026-10-02T13:21:15+00:00", + 401 + ], + [ + "2026-10-02T13:21:16+00:00", + 401 + ], + [ + "2026-10-02T13:21:16+00:00", + 401 + ], + [ + "2026-10-02T13:21:17+00:00", + 401 + ], + [ + "2026-10-02T13:21:17+00:00", + 401 + ], + [ + "2026-10-02T13:21:17+00:00", + 401 + ], + [ + "2026-10-02T13:21:18+00:00", + 401 + ], + [ + "2026-10-02T13:21:18+00:00", + 401 + ], + [ + "2026-10-02T13:21:19+00:00", + 401 + ], + [ + "2026-10-02T13:21:19+00:00", + 401 + ], + [ + "2026-10-02T13:21:19+00:00", + 401 + ], + [ + "2026-10-02T13:21:20+00:00", + 401 + ], + [ + "2026-10-02T13:21:20+00:00", + 401 + ], + [ + "2026-10-02T13:21:21+00:00", + 401 + ], + [ + "2026-10-02T13:21:21+00:00", + 401 + ], + [ + "2026-10-02T13:21:21+00:00", + 401 + ], + [ + "2026-10-02T13:21:22+00:00", + 401 + ], + [ + "2026-10-02T13:21:22+00:00", + 401 + ], + [ + "2026-10-02T13:21:23+00:00", + 401 + ], + [ + "2026-10-02T13:21:23+00:00", + 401 + ], + [ + "2026-10-02T13:21:23+00:00", + 401 + ], + [ + "2026-10-02T13:21:24+00:00", + 401 + ], + [ + "2026-10-02T13:21:24+00:00", + 401 + ], + [ + "2026-10-02T13:21:25+00:00", + 401 + ], + [ + "2026-10-02T13:21:25+00:00", + 401 + ], + [ + "2026-10-02T13:21:26+00:00", + 401 + ], + [ + "2026-10-02T13:21:26+00:00", + 401 + ], + [ + "2026-10-02T13:21:26+00:00", + 401 + ], + [ + "2026-10-02T13:21:27+00:00", + 401 + ], + [ + "2026-10-02T13:21:27+00:00", + 401 + ], + [ + "2026-10-02T13:21:28+00:00", + 401 + ], + [ + "2026-10-02T13:21:28+00:00", + 401 + ], + [ + "2026-10-02T13:21:28+00:00", + 401 + ], + [ + "2026-10-02T13:21:29+00:00", + 401 + ], + [ + "2026-10-02T13:21:29+00:00", + 401 + ], + [ + "2026-10-02T13:21:30+00:00", + 401 + ], + [ + "2026-10-02T13:21:30+00:00", + 401 + ], + [ + "2026-10-02T13:21:30+00:00", + 401 + ], + [ + "2026-10-02T13:21:31+00:00", + 401 + ], + [ + "2026-10-02T13:21:31+00:00", + 401 + ], + [ + "2026-10-02T13:21:32+00:00", + 401 + ], + [ + "2026-10-02T13:21:32+00:00", + 401 + ], + [ + "2026-10-02T13:21:32+00:00", + 401 + ], + [ + "2026-10-02T13:21:33+00:00", + 401 + ], + [ + "2026-10-02T13:21:33+00:00", + 401 + ], + [ + "2026-10-02T13:21:34+00:00", + 401 + ], + [ + "2026-10-02T13:21:34+00:00", + 401 + ], + [ + "2026-10-02T13:21:34+00:00", + 401 + ], + [ + "2026-10-02T13:21:35+00:00", + 401 + ], + [ + "2026-10-02T13:21:35+00:00", + 401 + ], + [ + "2026-10-02T13:21:36+00:00", + 401 + ], + [ + "2026-10-02T13:21:36+00:00", + 401 + ], + [ + "2026-10-02T13:21:36+00:00", + 401 + ], + [ + "2026-10-02T13:21:37+00:00", + 401 + ], + [ + "2026-10-02T13:21:37+00:00", + 401 + ], + [ + "2026-10-02T13:21:38+00:00", + 401 + ], + [ + "2026-10-02T13:21:38+00:00", + 401 + ], + [ + "2026-10-02T13:21:39+00:00", + 401 + ], + [ + "2026-10-02T13:21:39+00:00", + 401 + ], + [ + "2026-10-02T13:21:39+00:00", + 401 + ], + [ + "2026-10-02T13:21:40+00:00", + 401 + ], + [ + "2026-10-02T13:21:40+00:00", + 401 + ], + [ + "2026-10-02T13:21:41+00:00", + 401 + ], + [ + "2026-10-02T13:21:41+00:00", + 401 + ], + [ + "2026-10-02T13:21:41+00:00", + 401 + ], + [ + "2026-10-02T13:21:42+00:00", + 401 + ], + [ + "2026-10-02T13:21:42+00:00", + 401 + ], + [ + "2026-10-02T13:21:43+00:00", + 401 + ], + [ + "2026-10-02T13:21:43+00:00", + 401 + ], + [ + "2026-10-02T13:21:43+00:00", + 401 + ], + [ + "2026-10-02T13:21:44+00:00", + 401 + ], + [ + "2026-10-02T13:21:44+00:00", + 401 + ], + [ + "2026-10-02T13:21:45+00:00", + 401 + ], + [ + "2026-10-02T13:21:45+00:00", + 401 + ], + [ + "2026-10-02T13:21:45+00:00", + 401 + ], + [ + "2026-10-02T13:21:46+00:00", + 401 + ], + [ + "2026-10-02T13:21:46+00:00", + 401 + ], + [ + "2026-10-02T13:21:47+00:00", + 401 + ], + [ + "2026-10-02T13:21:47+00:00", + 401 + ], + [ + "2026-10-02T13:21:48+00:00", + 401 + ], + [ + "2026-10-02T13:21:48+00:00", + 401 + ], + [ + "2026-10-02T13:21:48+00:00", + 401 + ], + [ + "2026-10-02T13:21:49+00:00", + 401 + ], + [ + "2026-10-02T13:21:49+00:00", + 401 + ], + [ + "2026-10-02T13:21:50+00:00", + 401 + ] + ], + "planned_seconds": { + "outage": 180, + "post": 130, + "pre": 15 + }, + "prometheus_samples": [ + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:14+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:19+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:24+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:29+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:34+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:39+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.05263157894736842, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:44+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.10714285714285715, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "pending", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:49+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.23529411764705882, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "pending", + "Lab6HighHttpErrorRate": "pending", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:54+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "pending", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:59+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:05+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:10+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:15+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:20+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:25+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:30+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:35+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:40+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:45+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:50+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:55+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:00+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:05+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:10+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:15+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:20+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:25+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:30+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:35+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:40+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:45+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:50+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:55+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:00+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:05+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:10+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:15+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:20+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:25+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:30+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:35+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:40+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:45+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:50+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:55+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:00+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:05+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:10+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:15+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:20+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:25+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:30+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.8333333333333333, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:35+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.8, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:40+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.10714285714285715, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:45+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.07407407407407408, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:50+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0196078431372549, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:55+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:00+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:05+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:10+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:15+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:20+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:25+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:30+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:35+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:40+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + } + ], + "times": { + "db_healthy": "2026-10-02T13:19:32+00:00", + "db_stopped": "2026-10-02T13:16:30+00:00", + "restart_requested": "2026-10-02T13:19:30+00:00", + "started": "2026-10-02T13:16:14+00:00", + "stop_requested": "2026-10-02T13:16:29+00:00" + }, + "watch_errors": [] +} diff --git a/evidence/slo-quiet-baseline-2026-10-02.jsonl b/evidence/slo-quiet-baseline-2026-10-02.jsonl new file mode 100644 index 0000000..e6029c0 --- /dev/null +++ b/evidence/slo-quiet-baseline-2026-10-02.jsonl @@ -0,0 +1,6 @@ +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:28+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:33+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:38+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:43+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:48+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:53+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} diff --git a/evidence/slo-short-db-2026-10-02.json b/evidence/slo-short-db-2026-10-02.json new file mode 100644 index 0000000..8f6090e --- /dev/null +++ b/evidence/slo-short-db-2026-10-02.json @@ -0,0 +1,1584 @@ +{ + "case": "short", + "events": [ + [ + "2026-10-02T13:13:13+00:00", + 401 + ], + [ + "2026-10-02T13:13:13+00:00", + 401 + ], + [ + "2026-10-02T13:13:14+00:00", + 401 + ], + [ + "2026-10-02T13:13:14+00:00", + 401 + ], + [ + "2026-10-02T13:13:15+00:00", + 401 + ], + [ + "2026-10-02T13:13:15+00:00", + 401 + ], + [ + "2026-10-02T13:13:16+00:00", + 401 + ], + [ + "2026-10-02T13:13:16+00:00", + 401 + ], + [ + "2026-10-02T13:13:16+00:00", + 401 + ], + [ + "2026-10-02T13:13:17+00:00", + 401 + ], + [ + "2026-10-02T13:13:17+00:00", + 401 + ], + [ + "2026-10-02T13:13:18+00:00", + 401 + ], + [ + "2026-10-02T13:13:18+00:00", + 401 + ], + [ + "2026-10-02T13:13:18+00:00", + 401 + ], + [ + "2026-10-02T13:13:19+00:00", + 401 + ], + [ + "2026-10-02T13:13:19+00:00", + 401 + ], + [ + "2026-10-02T13:13:20+00:00", + 401 + ], + [ + "2026-10-02T13:13:20+00:00", + 401 + ], + [ + "2026-10-02T13:13:20+00:00", + 401 + ], + [ + "2026-10-02T13:13:21+00:00", + 401 + ], + [ + "2026-10-02T13:13:21+00:00", + 401 + ], + [ + "2026-10-02T13:13:22+00:00", + 401 + ], + [ + "2026-10-02T13:13:22+00:00", + 401 + ], + [ + "2026-10-02T13:13:23+00:00", + 401 + ], + [ + "2026-10-02T13:13:23+00:00", + 401 + ], + [ + "2026-10-02T13:13:23+00:00", + 401 + ], + [ + "2026-10-02T13:13:24+00:00", + 401 + ], + [ + "2026-10-02T13:13:24+00:00", + 401 + ], + [ + "2026-10-02T13:13:25+00:00", + 401 + ], + [ + "2026-10-02T13:13:25+00:00", + 401 + ], + [ + "2026-10-02T13:13:25+00:00", + 401 + ], + [ + "2026-10-02T13:13:26+00:00", + 401 + ], + [ + "2026-10-02T13:13:26+00:00", + 401 + ], + [ + "2026-10-02T13:13:27+00:00", + 401 + ], + [ + "2026-10-02T13:13:27+00:00", + 401 + ], + [ + "2026-10-02T13:13:27+00:00", + 401 + ], + [ + "2026-10-02T13:13:28+00:00", + 401 + ], + [ + "2026-10-02T13:13:33+00:00", + 503 + ], + [ + "2026-10-02T13:13:39+00:00", + 503 + ], + [ + "2026-10-02T13:13:44+00:00", + 503 + ], + [ + "2026-10-02T13:13:49+00:00", + 503 + ], + [ + "2026-10-02T13:13:55+00:00", + 503 + ], + [ + "2026-10-02T13:14:00+00:00", + 503 + ], + [ + "2026-10-02T13:14:06+00:00", + 503 + ], + [ + "2026-10-02T13:14:11+00:00", + 503 + ], + [ + "2026-10-02T13:14:16+00:00", + 503 + ], + [ + "2026-10-02T13:14:22+00:00", + 503 + ], + [ + "2026-10-02T13:14:27+00:00", + 503 + ], + [ + "2026-10-02T13:14:33+00:00", + 503 + ], + [ + "2026-10-02T13:14:35+00:00", + 401 + ], + [ + "2026-10-02T13:14:35+00:00", + 401 + ], + [ + "2026-10-02T13:14:35+00:00", + 401 + ], + [ + "2026-10-02T13:14:36+00:00", + 401 + ], + [ + "2026-10-02T13:14:36+00:00", + 401 + ], + [ + "2026-10-02T13:14:37+00:00", + 401 + ], + [ + "2026-10-02T13:14:37+00:00", + 401 + ], + [ + "2026-10-02T13:14:37+00:00", + 401 + ], + [ + "2026-10-02T13:14:38+00:00", + 401 + ], + [ + "2026-10-02T13:14:38+00:00", + 401 + ], + [ + "2026-10-02T13:14:39+00:00", + 401 + ], + [ + "2026-10-02T13:14:39+00:00", + 401 + ], + [ + "2026-10-02T13:14:39+00:00", + 401 + ], + [ + "2026-10-02T13:14:40+00:00", + 401 + ], + [ + "2026-10-02T13:14:40+00:00", + 401 + ], + [ + "2026-10-02T13:14:41+00:00", + 401 + ], + [ + "2026-10-02T13:14:41+00:00", + 401 + ], + [ + "2026-10-02T13:14:41+00:00", + 401 + ], + [ + "2026-10-02T13:14:42+00:00", + 401 + ], + [ + "2026-10-02T13:14:42+00:00", + 401 + ], + [ + "2026-10-02T13:14:43+00:00", + 401 + ], + [ + "2026-10-02T13:14:43+00:00", + 401 + ], + [ + "2026-10-02T13:14:43+00:00", + 401 + ], + [ + "2026-10-02T13:14:44+00:00", + 401 + ], + [ + "2026-10-02T13:14:44+00:00", + 401 + ], + [ + "2026-10-02T13:14:45+00:00", + 401 + ], + [ + "2026-10-02T13:14:45+00:00", + 401 + ], + [ + "2026-10-02T13:14:45+00:00", + 401 + ], + [ + "2026-10-02T13:14:46+00:00", + 401 + ], + [ + "2026-10-02T13:14:46+00:00", + 401 + ], + [ + "2026-10-02T13:14:47+00:00", + 401 + ], + [ + "2026-10-02T13:14:47+00:00", + 401 + ], + [ + "2026-10-02T13:14:48+00:00", + 401 + ], + [ + "2026-10-02T13:14:48+00:00", + 401 + ], + [ + "2026-10-02T13:14:48+00:00", + 401 + ], + [ + "2026-10-02T13:14:49+00:00", + 401 + ], + [ + "2026-10-02T13:14:49+00:00", + 401 + ], + [ + "2026-10-02T13:14:50+00:00", + 401 + ], + [ + "2026-10-02T13:14:50+00:00", + 401 + ], + [ + "2026-10-02T13:14:50+00:00", + 401 + ], + [ + "2026-10-02T13:14:51+00:00", + 401 + ], + [ + "2026-10-02T13:14:51+00:00", + 401 + ], + [ + "2026-10-02T13:14:52+00:00", + 401 + ], + [ + "2026-10-02T13:14:52+00:00", + 401 + ], + [ + "2026-10-02T13:14:52+00:00", + 401 + ], + [ + "2026-10-02T13:14:53+00:00", + 401 + ], + [ + "2026-10-02T13:14:53+00:00", + 401 + ], + [ + "2026-10-02T13:14:54+00:00", + 401 + ], + [ + "2026-10-02T13:14:54+00:00", + 401 + ], + [ + "2026-10-02T13:14:54+00:00", + 401 + ], + [ + "2026-10-02T13:14:55+00:00", + 401 + ], + [ + "2026-10-02T13:14:55+00:00", + 401 + ], + [ + "2026-10-02T13:14:56+00:00", + 401 + ], + [ + "2026-10-02T13:14:56+00:00", + 401 + ], + [ + "2026-10-02T13:14:56+00:00", + 401 + ], + [ + "2026-10-02T13:14:57+00:00", + 401 + ], + [ + "2026-10-02T13:14:57+00:00", + 401 + ], + [ + "2026-10-02T13:14:58+00:00", + 401 + ], + [ + "2026-10-02T13:14:58+00:00", + 401 + ], + [ + "2026-10-02T13:14:59+00:00", + 401 + ], + [ + "2026-10-02T13:14:59+00:00", + 401 + ], + [ + "2026-10-02T13:14:59+00:00", + 401 + ], + [ + "2026-10-02T13:15:00+00:00", + 401 + ], + [ + "2026-10-02T13:15:00+00:00", + 401 + ], + [ + "2026-10-02T13:15:01+00:00", + 401 + ], + [ + "2026-10-02T13:15:01+00:00", + 401 + ], + [ + "2026-10-02T13:15:01+00:00", + 401 + ], + [ + "2026-10-02T13:15:02+00:00", + 401 + ], + [ + "2026-10-02T13:15:02+00:00", + 401 + ], + [ + "2026-10-02T13:15:03+00:00", + 401 + ], + [ + "2026-10-02T13:15:03+00:00", + 401 + ], + [ + "2026-10-02T13:15:03+00:00", + 401 + ], + [ + "2026-10-02T13:15:04+00:00", + 401 + ], + [ + "2026-10-02T13:15:04+00:00", + 401 + ], + [ + "2026-10-02T13:15:05+00:00", + 401 + ], + [ + "2026-10-02T13:15:05+00:00", + 401 + ], + [ + "2026-10-02T13:15:05+00:00", + 401 + ], + [ + "2026-10-02T13:15:06+00:00", + 401 + ], + [ + "2026-10-02T13:15:06+00:00", + 401 + ], + [ + "2026-10-02T13:15:07+00:00", + 401 + ], + [ + "2026-10-02T13:15:07+00:00", + 401 + ], + [ + "2026-10-02T13:15:07+00:00", + 401 + ], + [ + "2026-10-02T13:15:08+00:00", + 401 + ], + [ + "2026-10-02T13:15:08+00:00", + 401 + ], + [ + "2026-10-02T13:15:09+00:00", + 401 + ], + [ + "2026-10-02T13:15:09+00:00", + 401 + ], + [ + "2026-10-02T13:15:09+00:00", + 401 + ], + [ + "2026-10-02T13:15:10+00:00", + 401 + ], + [ + "2026-10-02T13:15:10+00:00", + 401 + ], + [ + "2026-10-02T13:15:11+00:00", + 401 + ], + [ + "2026-10-02T13:15:11+00:00", + 401 + ], + [ + "2026-10-02T13:15:11+00:00", + 401 + ], + [ + "2026-10-02T13:15:12+00:00", + 401 + ], + [ + "2026-10-02T13:15:12+00:00", + 401 + ], + [ + "2026-10-02T13:15:13+00:00", + 401 + ], + [ + "2026-10-02T13:15:13+00:00", + 401 + ], + [ + "2026-10-02T13:15:14+00:00", + 401 + ], + [ + "2026-10-02T13:15:14+00:00", + 401 + ], + [ + "2026-10-02T13:15:14+00:00", + 401 + ], + [ + "2026-10-02T13:15:15+00:00", + 401 + ], + [ + "2026-10-02T13:15:15+00:00", + 401 + ], + [ + "2026-10-02T13:15:16+00:00", + 401 + ], + [ + "2026-10-02T13:15:16+00:00", + 401 + ], + [ + "2026-10-02T13:15:16+00:00", + 401 + ], + [ + "2026-10-02T13:15:17+00:00", + 401 + ], + [ + "2026-10-02T13:15:17+00:00", + 401 + ], + [ + "2026-10-02T13:15:18+00:00", + 401 + ], + [ + "2026-10-02T13:15:18+00:00", + 401 + ], + [ + "2026-10-02T13:15:18+00:00", + 401 + ], + [ + "2026-10-02T13:15:19+00:00", + 401 + ], + [ + "2026-10-02T13:15:19+00:00", + 401 + ], + [ + "2026-10-02T13:15:20+00:00", + 401 + ], + [ + "2026-10-02T13:15:20+00:00", + 401 + ], + [ + "2026-10-02T13:15:20+00:00", + 401 + ], + [ + "2026-10-02T13:15:21+00:00", + 401 + ], + [ + "2026-10-02T13:15:21+00:00", + 401 + ], + [ + "2026-10-02T13:15:22+00:00", + 401 + ], + [ + "2026-10-02T13:15:22+00:00", + 401 + ], + [ + "2026-10-02T13:15:22+00:00", + 401 + ], + [ + "2026-10-02T13:15:23+00:00", + 401 + ], + [ + "2026-10-02T13:15:23+00:00", + 401 + ], + [ + "2026-10-02T13:15:24+00:00", + 401 + ], + [ + "2026-10-02T13:15:24+00:00", + 401 + ], + [ + "2026-10-02T13:15:25+00:00", + 401 + ], + [ + "2026-10-02T13:15:25+00:00", + 401 + ], + [ + "2026-10-02T13:15:25+00:00", + 401 + ], + [ + "2026-10-02T13:15:26+00:00", + 401 + ], + [ + "2026-10-02T13:15:26+00:00", + 401 + ], + [ + "2026-10-02T13:15:27+00:00", + 401 + ], + [ + "2026-10-02T13:15:27+00:00", + 401 + ], + [ + "2026-10-02T13:15:27+00:00", + 401 + ], + [ + "2026-10-02T13:15:28+00:00", + 401 + ], + [ + "2026-10-02T13:15:28+00:00", + 401 + ], + [ + "2026-10-02T13:15:29+00:00", + 401 + ], + [ + "2026-10-02T13:15:29+00:00", + 401 + ], + [ + "2026-10-02T13:15:29+00:00", + 401 + ], + [ + "2026-10-02T13:15:30+00:00", + 401 + ], + [ + "2026-10-02T13:15:30+00:00", + 401 + ], + [ + "2026-10-02T13:15:31+00:00", + 401 + ], + [ + "2026-10-02T13:15:31+00:00", + 401 + ], + [ + "2026-10-02T13:15:31+00:00", + 401 + ], + [ + "2026-10-02T13:15:32+00:00", + 401 + ], + [ + "2026-10-02T13:15:32+00:00", + 401 + ], + [ + "2026-10-02T13:15:33+00:00", + 401 + ], + [ + "2026-10-02T13:15:33+00:00", + 401 + ], + [ + "2026-10-02T13:15:33+00:00", + 401 + ], + [ + "2026-10-02T13:15:34+00:00", + 401 + ], + [ + "2026-10-02T13:15:34+00:00", + 401 + ], + [ + "2026-10-02T13:15:35+00:00", + 401 + ], + [ + "2026-10-02T13:15:35+00:00", + 401 + ], + [ + "2026-10-02T13:15:35+00:00", + 401 + ], + [ + "2026-10-02T13:15:36+00:00", + 401 + ], + [ + "2026-10-02T13:15:36+00:00", + 401 + ], + [ + "2026-10-02T13:15:37+00:00", + 401 + ], + [ + "2026-10-02T13:15:37+00:00", + 401 + ], + [ + "2026-10-02T13:15:38+00:00", + 401 + ], + [ + "2026-10-02T13:15:38+00:00", + 401 + ], + [ + "2026-10-02T13:15:38+00:00", + 401 + ], + [ + "2026-10-02T13:15:39+00:00", + 401 + ], + [ + "2026-10-02T13:15:39+00:00", + 401 + ], + [ + "2026-10-02T13:15:40+00:00", + 401 + ], + [ + "2026-10-02T13:15:40+00:00", + 401 + ], + [ + "2026-10-02T13:15:40+00:00", + 401 + ], + [ + "2026-10-02T13:15:41+00:00", + 401 + ], + [ + "2026-10-02T13:15:41+00:00", + 401 + ], + [ + "2026-10-02T13:15:42+00:00", + 401 + ], + [ + "2026-10-02T13:15:42+00:00", + 401 + ], + [ + "2026-10-02T13:15:42+00:00", + 401 + ], + [ + "2026-10-02T13:15:43+00:00", + 401 + ], + [ + "2026-10-02T13:15:43+00:00", + 401 + ], + [ + "2026-10-02T13:15:44+00:00", + 401 + ], + [ + "2026-10-02T13:15:44+00:00", + 401 + ], + [ + "2026-10-02T13:15:44+00:00", + 401 + ], + [ + "2026-10-02T13:15:45+00:00", + 401 + ], + [ + "2026-10-02T13:15:45+00:00", + 401 + ], + [ + "2026-10-02T13:15:46+00:00", + 401 + ], + [ + "2026-10-02T13:15:46+00:00", + 401 + ], + [ + "2026-10-02T13:15:46+00:00", + 401 + ], + [ + "2026-10-02T13:15:47+00:00", + 401 + ], + [ + "2026-10-02T13:15:47+00:00", + 401 + ], + [ + "2026-10-02T13:15:48+00:00", + 401 + ], + [ + "2026-10-02T13:15:48+00:00", + 401 + ], + [ + "2026-10-02T13:15:48+00:00", + 401 + ], + [ + "2026-10-02T13:15:49+00:00", + 401 + ], + [ + "2026-10-02T13:15:49+00:00", + 401 + ], + [ + "2026-10-02T13:15:50+00:00", + 401 + ], + [ + "2026-10-02T13:15:50+00:00", + 401 + ], + [ + "2026-10-02T13:15:51+00:00", + 401 + ], + [ + "2026-10-02T13:15:51+00:00", + 401 + ], + [ + "2026-10-02T13:15:51+00:00", + 401 + ], + [ + "2026-10-02T13:15:52+00:00", + 401 + ], + [ + "2026-10-02T13:15:52+00:00", + 401 + ], + [ + "2026-10-02T13:15:53+00:00", + 401 + ], + [ + "2026-10-02T13:15:53+00:00", + 401 + ], + [ + "2026-10-02T13:15:53+00:00", + 401 + ], + [ + "2026-10-02T13:15:54+00:00", + 401 + ], + [ + "2026-10-02T13:15:54+00:00", + 401 + ], + [ + "2026-10-02T13:15:55+00:00", + 401 + ], + [ + "2026-10-02T13:15:55+00:00", + 401 + ], + [ + "2026-10-02T13:15:55+00:00", + 401 + ], + [ + "2026-10-02T13:15:56+00:00", + 401 + ], + [ + "2026-10-02T13:15:56+00:00", + 401 + ], + [ + "2026-10-02T13:15:57+00:00", + 401 + ], + [ + "2026-10-02T13:15:57+00:00", + 401 + ], + [ + "2026-10-02T13:15:57+00:00", + 401 + ], + [ + "2026-10-02T13:15:58+00:00", + 401 + ] + ], + "planned_seconds": { + "outage": 50, + "post": 90, + "pre": 15 + }, + "prometheus_samples": [ + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:13+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:18+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:23+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:28+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:33+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:38+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:43+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.05239864559019707, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:48+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.10986596449397801, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "pending", + "Lab6HighHttpErrorRate": "pending", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:53+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.1914814703057944, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "pending", + "Lab6HighHttpErrorRate": "pending", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:58+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:03+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:08+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:13+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:18+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:23+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:28+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:33+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:38+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:43+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.25, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:48+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.1111111111111111, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:53+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.05263157894736842, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:58+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.02, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:03+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:08+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:13+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:18+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:23+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:28+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:33+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:38+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:43+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:48+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + } + ], + "times": { + "db_healthy": "2026-10-02T13:14:21+00:00", + "db_stopped": "2026-10-02T13:13:28+00:00", + "restart_requested": "2026-10-02T13:14:18+00:00", + "started": "2026-10-02T13:13:13+00:00", + "stop_requested": "2026-10-02T13:13:28+00:00" + }, + "watch_errors": [] +} diff --git a/evidence/telemetry-backend-outage-2026-10-02.json b/evidence/telemetry-backend-outage-2026-10-02.json new file mode 100644 index 0000000..3ead1ad --- /dev/null +++ b/evidence/telemetry-backend-outage-2026-10-02.json @@ -0,0 +1,150 @@ +{ + "collector_counter_delta": { + "otelcol_exporter_enqueue_failed_spans": 19486.0, + "otelcol_exporter_send_failed_log_records": 11063.0, + "otelcol_exporter_send_failed_metric_points": 484.0, + "otelcol_exporter_send_failed_spans": 16547.0, + "otelcol_receiver_refused_log_records": 0.0, + "otelcol_receiver_refused_metric_points": 0.0, + "otelcol_receiver_refused_spans": 0.0 + }, + "collector_counters_after": { + "otelcol_exporter_enqueue_failed_spans": 112097.0, + "otelcol_exporter_send_failed_log_records": 31979.0, + "otelcol_exporter_send_failed_metric_points": 1570.0, + "otelcol_exporter_send_failed_spans": 44902.0, + "otelcol_receiver_refused_log_records": 0.0, + "otelcol_receiver_refused_metric_points": 0.0, + "otelcol_receiver_refused_spans": 0.0 + }, + "collector_counters_before": { + "otelcol_exporter_enqueue_failed_spans": 92611.0, + "otelcol_exporter_send_failed_log_records": 20916.0, + "otelcol_exporter_send_failed_metric_points": 1086.0, + "otelcol_exporter_send_failed_spans": 28355.0, + "otelcol_receiver_refused_log_records": 0.0, + "otelcol_receiver_refused_metric_points": 0.0, + "otelcol_receiver_refused_spans": 0.0 + }, + "load_passed": true, + "smoke_passed": true, + "smoke_tail": [ + "python3 tests/smoke.py --base-url \"http://127.0.0.1:8080\" ", + "PASS: lab6 public API scenario; infrastructure and grading evidence remain separate." + ], + "started": "2026-10-02T14:01:35+00:00", + "stats_after_35s": [ + { + "BlockIO": "2.23GB / 0B", + "CPUPerc": "1.21%", + "Container": "mtusi2026lab6-otel-collector-1", + "ID": "3bedc51054f5", + "MemPerc": "30.20%", + "MemUsage": "116MiB / 384MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "126MB / 37.1MB", + "PIDs": "10" + }, + { + "BlockIO": "40.6MB / 0B", + "CPUPerc": "14.25%", + "Container": "mtusi2026lab6-api-1", + "ID": "2ad72f3266ec", + "MemPerc": "32.39%", + "MemUsage": "82.91MiB / 256MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "22.6MB / 29.5MB", + "PIDs": "10" + }, + { + "BlockIO": "11.3MB / 0B", + "CPUPerc": "0.09%", + "Container": "mtusi2026lab6-worker-1", + "ID": "88e67b14961c", + "MemPerc": "7.12%", + "MemUsage": "54.67MiB / 768MiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "334kB / 453kB", + "PIDs": "8" + } + ], + "stats_before": [ + { + "BlockIO": "2.21GB / 0B", + "CPUPerc": "0.19%", + "Container": "mtusi2026lab6-otel-collector-1", + "ID": "3bedc51054f5", + "MemPerc": "18.44%", + "MemUsage": "70.82MiB / 384MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "110MB / 36.8MB", + "PIDs": "10" + }, + { + "BlockIO": "35.6MB / 0B", + "CPUPerc": "8.60%", + "Container": "mtusi2026lab6-api-1", + "ID": "2ad72f3266ec", + "MemPerc": "22.86%", + "MemUsage": "58.52MiB / 256MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "1.13MB / 1.78MB", + "PIDs": "10" + }, + { + "BlockIO": "10.8MB / 0B", + "CPUPerc": "0.15%", + "Container": "mtusi2026lab6-worker-1", + "ID": "88e67b14961c", + "MemPerc": "7.09%", + "MemUsage": "54.44MiB / 768MiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "188kB / 267kB", + "PIDs": "8" + } + ], + "stats_during": [ + { + "BlockIO": "2.23GB / 0B", + "CPUPerc": "2.40%", + "Container": "mtusi2026lab6-otel-collector-1", + "ID": "3bedc51054f5", + "MemPerc": "27.14%", + "MemUsage": "104.2MiB / 384MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "126MB / 37MB", + "PIDs": "10" + }, + { + "BlockIO": "40.6MB / 0B", + "CPUPerc": "7.68%", + "Container": "mtusi2026lab6-api-1", + "ID": "2ad72f3266ec", + "MemPerc": "32.65%", + "MemUsage": "83.58MiB / 256MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "22.5MB / 29.3MB", + "PIDs": "13" + }, + { + "BlockIO": "11.3MB / 0B", + "CPUPerc": "0.14%", + "Container": "mtusi2026lab6-worker-1", + "ID": "88e67b14961c", + "MemPerc": "7.12%", + "MemUsage": "54.67MiB / 768MiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "325kB / 431kB", + "PIDs": "8" + } + ], + "stores": [ + "prometheus", + "loki", + "tempo" + ], + "stores_healthy": true, + "stores_restored": true, + "stores_restored_at": "2026-10-02T14:03:01+00:00", + "stores_stopped": "2026-10-02T14:01:48+00:00" +} diff --git a/evidence/telemetry-outage-http-load-2026-10-02.json b/evidence/telemetry-outage-http-load-2026-10-02.json new file mode 100644 index 0000000..a81e211 --- /dev/null +++ b/evidence/telemetry-outage-http-load-2026-10-02.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.017, + "concurrency": 4, + "successful_requests": 17876, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 595.531, + "successful_latency_ms": { + "p50": 4.844, + "p95": 16.108, + "p99": 49.174 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/tempo-recovery-load-2026-10-02.json b/evidence/tempo-recovery-load-2026-10-02.json new file mode 100644 index 0000000..ec0c08d --- /dev/null +++ b/evidence/tempo-recovery-load-2026-10-02.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.011, + "concurrency": 4, + "successful_requests": 18860, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 628.445, + "successful_latency_ms": { + "p50": 4.677, + "p95": 14.469, + "p99": 44.625 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/scripts/measure-image-pipeline.py b/scripts/measure-image-pipeline.py index 6b27caa..4a744fe 100644 --- a/scripts/measure-image-pipeline.py +++ b/scripts/measure-image-pipeline.py @@ -41,6 +41,7 @@ def main(): parser.add_argument("--base-url", default="http://127.0.0.1:8080") parser.add_argument("--concurrency", type=int, default=4) parser.add_argument("--per-size", type=int, default=4) + parser.add_argument("--profile", choices=("small_320x200", "noise_800x600")) parser.add_argument("--output", required=True) args = parser.parse_args() if not 1 <= args.concurrency <= 16 or not 1 <= args.per_size <= 100: @@ -52,6 +53,8 @@ def main(): "small_320x200": (ROOT / "tests/fixtures/sample.png").read_bytes(), "noise_800x600": png_noise(800, 600), } + if args.profile: + images = {args.profile: images[args.profile]} items = [(name, data) for name, data in images.items() for _ in range(args.per_size)] def work(item): diff --git a/student-tests/test_observability_failure.py b/student-tests/test_observability_failure.py new file mode 100644 index 0000000..dd49af8 --- /dev/null +++ b/student-tests/test_observability_failure.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Prove business API survives a bounded outage of all telemetry stores.""" + +import json +import subprocess +import time +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlencode +from urllib.request import urlopen + + +ROOT = Path(__file__).resolve().parents[1] +FILES = ["-f", "compose.yaml"] +if (ROOT / "compose.local.yaml").exists(): + FILES += ["-f", "compose.local.yaml"] +FILES += ["-f", "compose.telemetry.yaml"] +COMPOSE = ["docker", "compose", *FILES, "--env-file", ".env"] +COUNTERS = ( + "otelcol_exporter_enqueue_failed_spans", + "otelcol_exporter_send_failed_spans", + "otelcol_exporter_send_failed_log_records", + "otelcol_exporter_send_failed_metric_points", + "otelcol_receiver_refused_spans", + "otelcol_receiver_refused_log_records", + "otelcol_receiver_refused_metric_points", +) + + +def run(*args): + return subprocess.run(args, cwd=ROOT, capture_output=True, text=True) + + +def stats(): + result = run("docker", "stats", "--no-stream", "--format", "{{json .}}", + "mtusi2026lab6-otel-collector-1", "mtusi2026lab6-api-1", "mtusi2026lab6-worker-1") + return [json.loads(line) for line in result.stdout.splitlines()] if result.returncode == 0 else [] + + +def metric(name): + url = "http://127.0.0.1:9090/api/v1/query?" + urlencode({"query": f"sum({name})"}) + try: + with urlopen(url, timeout=5) as response: + rows = json.load(response)["data"]["result"] + except (OSError, ValueError): + return None + return float(rows[0]["value"][1]) if rows else None + + +def stores_up(): + url = "http://127.0.0.1:9090/api/v1/query?" + urlencode({"query": "up{job=~\"prometheus|loki|tempo\"}"}) + try: + with urlopen(url, timeout=5) as response: + rows = json.load(response)["data"]["result"] + except (OSError, ValueError): + return False + return len(rows) == 3 and all(row["value"][1] == "1" for row in rows) + + +def main(): + output = ROOT / "evidence/telemetry-backend-outage-2026-10-02.json" + data = {"started": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "stores": ["prometheus", "loki", "tempo"], "stats_before": stats(), + "collector_counters_before": {name: metric(name) for name in COUNTERS}} + stopped = False + try: + result = run(*COMPOSE, "stop", "prometheus", "loki", "tempo") + if result.returncode: + raise RuntimeError(result.stderr[-500:]) + stopped = True + data["stores_stopped"] = datetime.now(timezone.utc).isoformat(timespec="seconds") + smoke = run("make", "test", "BASE_URL=http://127.0.0.1:8080") + data["smoke_passed"] = smoke.returncode == 0 + data["smoke_tail"] = (smoke.stdout + smoke.stderr).splitlines()[-5:] + load = run("python3", "tests/load.py", "--base-url", "http://127.0.0.1:8080", + "--duration", "30", "--concurrency", "4", + "--output", "evidence/telemetry-outage-http-load-2026-10-02.json") + data["load_passed"] = load.returncode == 0 + data["stats_during"] = stats() + # Longer than Collector's 30-second exporter retry horizon. + time.sleep(35) + data["stats_after_35s"] = stats() + finally: + if stopped: + restored = run(*COMPOSE, "up", "-d", "prometheus", "loki", "tempo") + data["stores_restored"] = restored.returncode == 0 + data["stores_restored_at"] = datetime.now(timezone.utc).isoformat(timespec="seconds") + healthy_since = None + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + if stores_up(): + healthy_since = healthy_since or time.monotonic() + if time.monotonic() - healthy_since >= 30: + break + else: + healthy_since = None + time.sleep(5) + data["stores_healthy"] = (healthy_since is not None and + time.monotonic() - healthy_since >= 30 and stores_up()) + data["collector_counters_after"] = {name: metric(name) for name in COUNTERS} + data["collector_counter_delta"] = { + name: (data["collector_counters_after"][name] - data["collector_counters_before"][name] + if data["collector_counters_before"][name] is not None and + data["collector_counters_after"][name] is not None else None) + for name in COUNTERS + } + output.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"smoke_passed": data.get("smoke_passed"), + "load_passed": data.get("load_passed"), + "stores_restored": data.get("stores_restored"), + "collector_counter_delta": data["collector_counter_delta"], + "output": str(output)}, sort_keys=True)) + if not all(data.get(key) for key in ("smoke_passed", "load_passed", "stores_restored", "stores_healthy")): + raise SystemExit("business path or telemetry recovery failed") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_queue_incident.py b/student-tests/test_queue_incident.py new file mode 100644 index 0000000..d112841 --- /dev/null +++ b/student-tests/test_queue_incident.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Stop worker, build a real durable backlog, then verify queue-lag recovery.""" + +import json +import subprocess +import sys +import threading +import time +from datetime import datetime, timezone +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +sys.path.insert(0, str(ROOT / "scripts")) +from client import Client # noqa: E402 +from slo_watch import snapshot # noqa: E402 + + +FILES = ["-f", "compose.yaml"] +if (ROOT / "compose.local.yaml").exists(): + FILES += ["-f", "compose.local.yaml"] +FILES += ["-f", "compose.telemetry.yaml"] + + +def compose(*args): + subprocess.run(["docker", "compose", *FILES, "--env-file", ".env", *args], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL) + + +def now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def main(): + client = Client("http://127.0.0.1:8080") + alice, _bob, _eve, chat = client.bootstrap() + image = (ROOT / "tests/fixtures/sample.png").read_bytes() + samples = [] + stop = threading.Event() + + def watch(): + while not stop.is_set(): + samples.append(snapshot("http://127.0.0.1:9090")) + stop.wait(5) + + watcher = threading.Thread(target=watch, daemon=True) + watcher.start() + times = {"started": now()} + worker_stopped = False + try: + compose("stop", "worker") + worker_stopped = True + times["worker_stopped"] = now() + attachments = [] + for _ in range(8): + response = client.call( + "POST", f"/api/v1/chats/{chat['id']}/attachments?filename=queue-probe.png", + 202, actor=alice, raw=image, headers={"Content-Type": "image/png"}, schema="Attachment", + ) + attachments.append(response["json"]) + times["uploads_accepted"] = now() + time.sleep(35) + compose("up", "-d", "worker") + worker_stopped = False + times["worker_started"] = now() + for item in attachments: + client.wait_ready(alice, item, wait_seconds=120) + times["all_ready"] = now() + time.sleep(40) + finally: + if worker_stopped: + compose("up", "-d", "worker") + stop.set() + watcher.join(timeout=6) + states = [item["alerts"].get("Lab6QueueLag") for item in samples] + data = {"times": times, "queued_uploads": len(attachments), + "ready_uploads": len(attachments), "prometheus_samples": samples, + "queue_alert_states": sorted(set(states))} + path = ROOT / "evidence/queue-worker-stop-2026-10-02.json" + path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"queued_uploads": len(attachments), "states": data["queue_alert_states"], + "times": times, "output": str(path)}, sort_keys=True)) + if "firing" not in states or states[-1] != "inactive": + raise SystemExit("queue lag alert did not fire and resolve") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_slo_alerts.py b/student-tests/test_slo_alerts.py new file mode 100644 index 0000000..2818276 --- /dev/null +++ b/student-tests/test_slo_alerts.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Reproduce a DB outage and capture Prometheus burn-rate transitions.""" + +import argparse +import json +import subprocess +import sys +import threading +import time +from datetime import datetime, timezone +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +COMPOSE_FILES = ["-f", "compose.yaml"] +if (ROOT / "compose.local.yaml").exists(): + COMPOSE_FILES += ["-f", "compose.local.yaml"] +COMPOSE_FILES += ["-f", "compose.telemetry.yaml"] +sys.path.insert(0, str(ROOT / "scripts")) +from slo_watch import snapshot # noqa: E402 + + +PROBE = r''' +import base64, json, sys, time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +end = time.monotonic() + int(sys.argv[1]) +events = [] +headers = {"Authorization": "Basic " + base64.b64encode(b"lab6_probe:synthetic-invalid-password").decode()} +def run_one(_): + result = [] + while time.monotonic() < end: + req = Request("http://127.0.0.1:8080/api/v1/auth/sessions", data=b"", headers=headers, method="POST") + try: + with urlopen(req, timeout=30) as response: + code = response.status + except HTTPError as exc: + code = exc.code + except OSError: + code = 0 + result.append([datetime.now(timezone.utc).isoformat(timespec="seconds"), code]) + time.sleep(0.4) + return result +with ThreadPoolExecutor(max_workers=1) as pool: + for batch in pool.map(run_one, range(1)): + events.extend(batch) +print(json.dumps({"events": sorted(events)})) +''' + + +def utc_now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def compose(*args): + subprocess.run( + ["docker", "compose", *COMPOSE_FILES, "--env-file", ".env", *args], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--case", choices=("short", "long"), required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + pre, outage, post = (15, 50, 90) if args.case == "short" else (15, 180, 130) + duration = pre + outage + post + 10 + samples, watch_errors, times = [], [], {"started": utc_now()} + stop_watching = threading.Event() + + def watch(): + while not stop_watching.is_set(): + try: + samples.append(snapshot("http://127.0.0.1:9090")) + except (OSError, ValueError, KeyError) as exc: + watch_errors.append({"timestamp": utc_now(), "type": type(exc).__name__}) + stop_watching.wait(5) + + watcher = threading.Thread(target=watch, daemon=True) + watcher.start() + command = [ + "docker", "compose", *COMPOSE_FILES, "--env-file", ".env", "exec", "-T", + "api", "python", "-u", "-", str(duration), + ] + probe = subprocess.Popen(command, cwd=ROOT, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + probe.stdin.write(PROBE) + probe.stdin.close() + db_stopped = False + try: + time.sleep(pre) + times["stop_requested"] = utc_now() + compose("stop", "postgres") + db_stopped = True + times["db_stopped"] = utc_now() + time.sleep(outage) + times["restart_requested"] = utc_now() + compose("up", "-d", "--wait", "postgres") + db_stopped = False + times["db_healthy"] = utc_now() + time.sleep(post) + finally: + if db_stopped: + compose("up", "-d", "--wait", "postgres") + times["db_healthy_after_cleanup"] = utc_now() + stop_watching.set() + watcher.join(timeout=6) + + output = probe.stdout.read() + error = probe.stderr.read() + probe.wait(timeout=15) + result = json.loads(output) if probe.returncode == 0 else {"error": error[-500:]} + result.update({"case": args.case, "planned_seconds": {"pre": pre, "outage": outage, "post": post}, + "times": times, "prometheus_samples": samples, "watch_errors": watch_errors}) + path = Path(args.output) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + states = {name: {sample["alerts"].get(name) for sample in samples} + for name in ("Lab6HighHttpErrorRate", "Lab6ApiBurnShort", "Lab6ApiBurnLong")} + codes = {event[1] for event in result.get("events", [])} + print(json.dumps({"case": args.case, "codes": sorted(codes), + "states": {name: sorted(values) for name, values in states.items()}, + "samples": len(samples), "output": str(path)}, sort_keys=True)) + expected = {"Lab6HighHttpErrorRate", "Lab6ApiBurnShort"} + if args.case == "long": + expected.add("Lab6ApiBurnLong") + if probe.returncode or not {401, 503} <= codes or any("firing" not in states[name] for name in expected): + raise SystemExit("incident did not demonstrate the expected HTTP SLO transitions") + if any(sample["alerts"].get(name) != "inactive" for name in expected for sample in samples[-3:]): + raise SystemExit("alerts did not resolve after database recovery") + + +if __name__ == "__main__": + main()