# 2026-11-14T16:52:15Z python3 student-tests/telemetry_outage.py --output evidence/telemetry-outage.json
[  43.46s] phase A_baseline done {"api": {"requests": 1708, "ok": 1708, "codes": {"200": 760, "201": 760, "202": 188}, "error_ratio_5xx": 0.0, "ok_rps": 40.1, "p50_ms": 8.1, "p95_ms": 20.9, "p99_ms": 46.1}, "mem": {"api-2": {"mem_mib": 44.5, "cpu": "8.81%"}, "api-1": {"mem_mib": 44.9, "cpu": "6.60%"}, "worker-1": {"mem_mib": 42.3, "cpu": "29.01%"}, "worker-2": {"mem_mib": 46.6, "cpu": "28.02%"}, "relay-1": {"mem_mib": 34.7, "cpu": "2.16%"}, "otel-collector-1": {"mem_mib": 77.7, "cpu": "0.94%"}}, "alerts": []}
[ 167.16s] phase B_tempo_loki_down done {"api": {"requests": 4998, "ok": 4998, "codes": {"200": 2221, "201": 2221, "202": 556}, "error_ratio_5xx": 0.0, "ok_rps": 40.5, "p50_ms": 6.9, "p95_ms": 19.5, "p99_ms": 37.5}, "mem": {"api-2": {"mem_mib": 44.6, "cpu": "6.04%"}, "api-1": {"mem_mib": 45.1, "cpu": "6.28%"}, "worker-1": {"mem_mib": 55.2, "cpu": "38.06%"}, "worker-2": {"mem_mib": 42.8, "cpu": "28.94%"}, "relay-1": {"mem_mib": 34.8, "cpu": "0.97%"}, "otel-collector-1": {"mem_mib": 104.7, "cpu": "1.14%"}}, "alerts": ["MessengerTelemetryBackendDown:firing:loki", "MessengerTelemetryBackendDown:firing:tempo"]}
[ 210.50s] phase C_backends_restored done {"api": {"requests": 1730, "ok": 1730, "codes": {"200": 769, "201": 769, "202": 192}, "error_ratio_5xx": 0.0, "ok_rps": 40.0, "p50_ms": 6.8, "p95_ms": 23.9, "p99_ms": 77.3}, "mem": {"api-2": {"mem_mib": 44.7, "cpu": "4.96%"}, "api-1": {"mem_mib": 45.2, "cpu": "4.86%"}, "worker-1": {"mem_mib": 42.7, "cpu": "34.46%"}, "worker-2": {"mem_mib": 42.8, "cpu": "23.68%"}, "relay-1": {"mem_mib": 34.8, "cpu": "0.81%"}, "otel-collector-1": {"mem_mib": 125.1, "cpu": "1.15%"}}, "alerts": ["MessengerTelemetryExportFailing:pending"]}
[ 210.58s] collector-side loss while Tempo+Loki were down (increase over B+C) {"spans_send_failed": 540, "spans_queue_full": 11379, "logs_send_failed": 119, "logs_queue_full": 1140, "spans_accepted": 43087, "logs_accepted": 8807, "peak_exporter_queue": {"otlp_grpc/tempo": 100.0, "otlp_http/loki": 100.0, "otlp_http/prometheus": 0.0}, "queue_capacity": 100.0, "peak_heap_mib": 100.5}
[ 303.72s] phase D_collector_down done {"api": {"requests": 3796, "ok": 3796, "codes": {"200": 1687, "201": 1687, "202": 422}, "error_ratio_5xx": 0.0, "ok_rps": 40.9, "p50_ms": 5.8, "p95_ms": 18.2, "p99_ms": 36.1}, "mem": {"api-2": {"mem_mib": 54.7, "cpu": "6.07%"}, "api-1": {"mem_mib": 53.1, "cpu": "5.22%"}, "worker-1": {"mem_mib": 46.7, "cpu": "31.66%"}, "worker-2": {"mem_mib": 47.5, "cpu": "31.51%"}, "relay-1": {"mem_mib": 36.3, "cpu": "0.84%"}}, "alerts": ["MessengerTelemetryBackendDown:firing:otel-collector", "MessengerTelemetryExportFailing:firing"]}
[ 351.50s] phase E_collector_restored done {"api": {"requests": 1970, "ok": 1970, "codes": {"200": 876, "201": 876, "202": 218}, "error_ratio_5xx": 0.0, "ok_rps": 41.3, "p50_ms": 5.4, "p95_ms": 12.1, "p99_ms": 26.5}, "mem": {"api-2": {"mem_mib": 51.0, "cpu": "7.56%"}, "api-1": {"mem_mib": 51.5, "cpu": "6.57%"}, "worker-1": {"mem_mib": 45.9, "cpu": "33.94%"}, "worker-2": {"mem_mib": 58.0, "cpu": "29.19%"}, "relay-1": {"mem_mib": 36.2, "cpu": "1.23%"}, "otel-collector-1": {"mem_mib": 69.8, "cpu": "2.53%"}}, "alerts": []}
[ 366.60s] SDK-side loss while the Collector was down (increase over D+E) {"sdk_spans_export_failed": 23518, "sdk_logs_export_failed": 4828, "sdk_spans_queue_full": 0, "sdk_logs_queue_full": 0, "sdk_spans_processed_ok": 37545, "sdk_export_error_types": ["ConnectionError"]}
[ 366.69s] telemetry counters after recovery {"collector_spans_send_failed": 0.0, "collector_spans_enqueue_failed": 0.0, "collector_logs_send_failed": 0.0, "collector_logs_enqueue_failed": 0.0, "collector_spans_sent": 13738.0, "collector_queue_tempo": 0.0, "collector_queue_loki": 0.0, "collector_rss_mib": 231.6, "collector_heap_mib": 35.7, "sdk_spans_dropped_queue_full": 0.0, "sdk_logs_dropped_queue_full": 0.0, "sdk_span_export_failed": 23518.0, "sdk_log_export_failed": 4828.0, "sdk_spans_processed_ok": 152482.0}
[ 366.70s] verdict {"api_5xx_total": 0, "max_mem_mib": {"api": 57.5, "worker": 68.3, "relay": 36.4, "otel-collector": 125.1}, "baseline_mem_mib": {"api": 44.9, "worker": 46.6, "relay": 34.7, "otel-collector": 77.7}, "loss": {"collector_while_backends_down": {"spans_send_failed": 540, "spans_queue_full": 11379, "logs_send_failed": 119, "logs_queue_full": 1140, "spans_accepted": 43087, "logs_accepted": 8807, "peak_exporter_queue": {"otlp_grpc/tempo": 100.0, "otlp_http/loki": 100.0, "otlp_http/prometheus": 0.0}, "queue_capacity": 100.0, "peak_heap_mib": 100.5}, "sdk_while_collector_down": {"sdk_spans_export_failed": 23518, "sdk_logs_export_failed": 4828, "sdk_spans_queue_full": 0, "sdk_logs_queue_full": 0, "sdk_spans_processed_ok": 37545, "sdk_export_error_types": ["ConnectionError"]}}}
PASS: telemetry outage: API 5xx=0, max RSS {'api': 57.5, 'worker': 68.3, 'relay': 36.4, 'otel-collector': 125.1}, losses {'collector_while_backends_down': {'spans_send_failed': 540, 'spans_queue_full': 11379, 'logs_send_failed': 119, 'logs_queue_full': 1140, 'spans_accepted': 43087, 'logs_accepted': 8807, 'peak_exporter_queue': {'otlp_grpc/tempo': 100.0, 'otlp_http/loki': 100.0, 'otlp_http/prometheus': 0.0}, 'queue_capacity': 100.0, 'peak_heap_mib': 100.5}, 'sdk_while_collector_down': {'sdk_spans_export_failed': 23518, 'sdk_logs_export_failed': 4828, 'sdk_spans_queue_full': 0, 'sdk_logs_queue_full': 0, 'sdk_spans_processed_ok': 37545, 'sdk_export_error_types': ['ConnectionError']}}
exit=0
