$ python -u research/incidents.py --out evidence\recheck_2026-11-12
{
  "incidents": [
    {
      "incident": "worker_down",
      "alert": "QueueLagGrowing",
      "injected_at_s": 25.0,
      "firing_at_s": 93.3,
      "detection_seconds": 68.3,
      "recovered_cmd_at_s": 96.5,
      "resolved_at_s": 118.3,
      "fired": true,
      "resolved": true,
      "evidence": {
        "queue_oldest_age_s": 60.743133,
        "worker_capacity_series_present": true,
        "jobs_ready_rate_1m": 0.0,
        "diagnosis": "возраст старейшей задачи растёт при нулевой скорости обработки jobs (worker не отдаёт метрики) -> остановлен worker, а не медленная БД",
        "alerts_in_state_firing": [
          "HighHttpErrorRate",
          "HighUploadErrorRate",
          "QueueLagGrowing"
        ]
      },
      "time_to_resolve_after_recovery_s": 21.8
    },
    {
      "incident": "slow_db",
      "alert": "HighLatencyP95",
      "injected_at_s": 124.3,
      "firing_at_s": 192.0,
      "detection_seconds": 67.7,
      "recovered_cmd_at_s": 193.7,
      "resolved_at_s": 237.2,
      "fired": true,
      "resolved": true,
      "evidence": {
        "p95_s": 0.7828124999999998,
        "db_pool_waiting": 0.0,
        "slow_traces_with_long_db_span": 20,
        "slow_http_post_messages_traces": 20,
        "diagnosis": "p95 высокий, а время в спанах db.connection сопоставимо с длительностью запроса и растёт ожидание пула -> медленная БД, а не обработчик",
        "alerts_in_state_firing": [
          "HighLatencyP95"
        ]
      },
      "time_to_resolve_after_recovery_s": 43.5
    },
    {
      "incident": "s3_down",
      "alert": "HighUploadErrorRate",
      "injected_at_s": 243.2,
      "firing_at_s": 327.3,
      "detection_seconds": 84.1,
      "recovered_cmd_at_s": 328.6,
      "resolved_at_s": 367.9,
      "fired": true,
      "resolved": true,
      "evidence": {
        "upload_error_ratio_1m": 1.0,
        "global_error_ratio_1m": 0.011166945840312675,
        "error_traces": 20,
        "s3_put_failed_log_lines": 37,
        "diagnosis": "рост доли 5xx на маршруте загрузки при нормальных остальных; спаны s3.put_object с ошибкой и логи s3_put_failed -> недоступен S3",
        "alerts_in_state_firing": [
          "HighUploadErrorRate",
          "QueueLagGrowing"
        ]
      },
      "time_to_resolve_after_recovery_s": 39.3
    },
    {
      "incident": "telemetry_backend",
      "alert": "TelemetryExportFailing",
      "injected_at_s": 373.9,
      "firing_at_s": 442.9,
      "detection_seconds": 69.1,
      "recovered_cmd_at_s": 444.7,
      "resolved_at_s": 512.2,
      "fired": true,
      "resolved": true,
      "evidence": {
        "api_error_ratio_1m": 0.0,
        "api_rps": 62.62045892577912,
        "collector_queue_size": 32.0,
        "collector_send_failed_spans_rate": 62.94195947803441,
        "collector_rss_mib": 239.0,
        "diagnosis": "ошибки экспорта растут, очередь ограничена; бизнес-метрики API без роста ошибок -> сбой хранилища телеметрии, не приложения",
        "alerts_in_state_firing": [
          "TelemetryExportFailing"
        ]
      },
      "time_to_resolve_after_recovery_s": 67.5
    }
  ],
  "traffic": {
    "msg_2": 15471,
    "up_2": 10598,
    "up_5": 2175,
    "up_err": 45
  }
}
exit=0

$ python -u research/obs_cost.py --out evidence\recheck_2026-11-12 --profiles off on100 head10 tail --repeat 2
{"profile": "off", "post": {"requests": 41303, "ok": 40440, "errors": {"500": 863}, "seconds": 30, "ok_rps": 1348.0, "error_ratio": 0.0209, "p50_ms": 17.39, "p95_ms": 74.05, "p99_ms": 618.63}, "docker_stats": {"worker": {"avg_cpu_percent": 1.6, "peak_mem_mib": 54.6}, "api": {"avg_cpu_percent": 50.6, "peak_mem_mib": 76.4}, "otel-collector": {"avg_cpu_percent": 0.1, "peak_mem_mib": 45.1}, "tempo": {"avg_cpu_percent": 0.3, "peak_mem_mib": 426.2}, "lb": {"avg_cpu_percent": 23.0, "peak_mem_mib": 18.3}, "grafana": {"avg_cpu_percent": 1.1, "peak_mem_mib": 314.4}, "rabbitmq": {"avg_cpu_percent": 14.5, "peak_mem_mib": 177.9}, "postgres": {"avg_cpu_percent": 95.9, "peak_mem_mib": 264.3}, "loki": {"avg_cpu_percent": 0.5, "peak_mem_mib": 119.8}, "seaweedfs": {"avg_cpu_percent": 1.3, "peak_mem_mib": 524.7}, "prometheus": {"avg_cpu_percent": 0.7, "peak_mem_mib": 75.6}, "valkey": {"avg_cpu_percent": 4.7, "peak_mem_mib": 10.7}}, "collector_delta": {"receiver_accepted_spans": 0, "receiver_accepted_log_records": 0, "receiver_accepted_metric_points": 0, "receiver_refused_spans": 0, "exporter_sent_spans": 0, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 0, "loki_bytes_delta": 0, "errors_injected_500": 873, "slow_requests_gt500ms": 823, "traces_in_tempo": {"all_http": 0, "error": 0, "slow_gt500ms": 0}, "kept_ratio": {"errors": 0.0, "slow": 0.0}}
{"profile": "off", "post": {"requests": 42499, "ok": 41641, "errors": {"500": 858}, "seconds": 30, "ok_rps": 1388.0, "error_ratio": 0.0202, "p50_ms": 16.03, "p95_ms": 72.03, "p99_ms": 617.51}, "docker_stats": {"worker": {"avg_cpu_percent": 1.4, "peak_mem_mib": 54.6}, "api": {"avg_cpu_percent": 49.7, "peak_mem_mib": 64.0}, "otel-collector": {"avg_cpu_percent": 0.1, "peak_mem_mib": 43.7}, "tempo": {"avg_cpu_percent": 0.3, "peak_mem_mib": 388.5}, "lb": {"avg_cpu_percent": 22.6, "peak_mem_mib": 21.8}, "grafana": {"avg_cpu_percent": 1.4, "peak_mem_mib": 315.2}, "rabbitmq": {"avg_cpu_percent": 8.3, "peak_mem_mib": 178.2}, "postgres": {"avg_cpu_percent": 94.0, "peak_mem_mib": 263.0}, "loki": {"avg_cpu_percent": 0.5, "peak_mem_mib": 121.8}, "seaweedfs": {"avg_cpu_percent": 1.4, "peak_mem_mib": 505.8}, "prometheus": {"avg_cpu_percent": 0.6, "peak_mem_mib": 60.9}, "valkey": {"avg_cpu_percent": 4.5, "peak_mem_mib": 10.7}}, "collector_delta": {"receiver_accepted_spans": 0, "receiver_accepted_log_records": 0, "receiver_accepted_metric_points": 0, "receiver_refused_spans": 0, "exporter_sent_spans": 0, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 0, "loki_bytes_delta": 0, "errors_injected_500": 873, "slow_requests_gt500ms": 888, "traces_in_tempo": {"all_http": 0, "error": 0, "slow_gt500ms": 0}, "kept_ratio": {"errors": 0.0, "slow": 0.0}}
{"profile": "on100", "post": {"requests": 38267, "ok": 37530, "errors": {"500": 737}, "seconds": 30, "ok_rps": 1251.0, "error_ratio": 0.0193, "p50_ms": 24.79, "p95_ms": 97.91, "p99_ms": 631.54}, "docker_stats": {"worker": {"avg_cpu_percent": 2.2, "peak_mem_mib": 64.5}, "api": {"avg_cpu_percent": 80.6, "peak_mem_mib": 83.2}, "otel-collector": {"avg_cpu_percent": 3.4, "peak_mem_mib": 75.0}, "tempo": {"avg_cpu_percent": 11.2, "peak_mem_mib": 745.7}, "lb": {"avg_cpu_percent": 20.2, "peak_mem_mib": 21.5}, "grafana": {"avg_cpu_percent": 1.1, "peak_mem_mib": 320.0}, "rabbitmq": {"avg_cpu_percent": 12.1, "peak_mem_mib": 157.0}, "postgres": {"avg_cpu_percent": 92.6, "peak_mem_mib": 282.2}, "loki": {"avg_cpu_percent": 4.4, "peak_mem_mib": 194.8}, "seaweedfs": {"avg_cpu_percent": 1.4, "peak_mem_mib": 494.7}, "prometheus": {"avg_cpu_percent": 0.5, "peak_mem_mib": 60.8}, "valkey": {"avg_cpu_percent": 5.4, "peak_mem_mib": 11.5}}, "collector_delta": {"receiver_accepted_spans": 90462, "receiver_accepted_log_records": 44825, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 90462, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 47091712, "loki_bytes_delta": 30777344, "errors_injected_500": 761, "slow_requests_gt500ms": 808, "traces_in_tempo": {"all_http": 5000, "error": 848, "slow_gt500ms": 902}, "kept_ratio": {"errors": 1.114, "slow": 1.116}}
{"profile": "on100", "post": {"requests": 37970, "ok": 37178, "errors": {"500": 792}, "seconds": 30, "ok_rps": 1239.3, "error_ratio": 0.0209, "p50_ms": 24.47, "p95_ms": 102.73, "p99_ms": 627.88}, "docker_stats": {"worker": {"avg_cpu_percent": 1.7, "peak_mem_mib": 60.3}, "api": {"avg_cpu_percent": 78.4, "peak_mem_mib": 75.4}, "otel-collector": {"avg_cpu_percent": 3.3, "peak_mem_mib": 72.0}, "tempo": {"avg_cpu_percent": 7.9, "peak_mem_mib": 526.0}, "lb": {"avg_cpu_percent": 19.5, "peak_mem_mib": 21.7}, "grafana": {"avg_cpu_percent": 0.8, "peak_mem_mib": 321.6}, "rabbitmq": {"avg_cpu_percent": 12.0, "peak_mem_mib": 188.4}, "postgres": {"avg_cpu_percent": 89.5, "peak_mem_mib": 284.4}, "loki": {"avg_cpu_percent": 4.8, "peak_mem_mib": 216.5}, "seaweedfs": {"avg_cpu_percent": 1.3, "peak_mem_mib": 494.4}, "prometheus": {"avg_cpu_percent": 0.7, "peak_mem_mib": 61.0}, "valkey": {"avg_cpu_percent": 5.0, "peak_mem_mib": 11.5}}, "collector_delta": {"receiver_accepted_spans": 89310, "receiver_accepted_log_records": 43824, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 89310, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 22323200, "loki_bytes_delta": 30490624, "errors_injected_500": 800, "slow_requests_gt500ms": 737, "traces_in_tempo": {"all_http": 5000, "error": 881, "slow_gt500ms": 850}, "kept_ratio": {"errors": 1.101, "slow": 1.153}}
{"profile": "head10", "post": {"requests": 34821, "ok": 34102, "errors": {"500": 719}, "seconds": 30, "ok_rps": 1136.7, "error_ratio": 0.0206, "p50_ms": 26.35, "p95_ms": 123.14, "p99_ms": 630.88}, "docker_stats": {"worker": {"avg_cpu_percent": 2.0, "peak_mem_mib": 62.5}, "api": {"avg_cpu_percent": 67.6, "peak_mem_mib": 83.7}, "otel-collector": {"avg_cpu_percent": 1.9, "peak_mem_mib": 68.4}, "tempo": {"avg_cpu_percent": 0.6, "peak_mem_mib": 472.9}, "lb": {"avg_cpu_percent": 19.2, "peak_mem_mib": 23.2}, "grafana": {"avg_cpu_percent": 0.9, "peak_mem_mib": 322.9}, "rabbitmq": {"avg_cpu_percent": 10.5, "peak_mem_mib": 194.3}, "postgres": {"avg_cpu_percent": 89.7, "peak_mem_mib": 302.1}, "loki": {"avg_cpu_percent": 4.8, "peak_mem_mib": 239.9}, "seaweedfs": {"avg_cpu_percent": 1.5, "peak_mem_mib": 494.4}, "prometheus": {"avg_cpu_percent": 0.6, "peak_mem_mib": 60.9}, "valkey": {"avg_cpu_percent": 5.1, "peak_mem_mib": 12.8}}, "collector_delta": {"receiver_accepted_spans": 8506, "receiver_accepted_log_records": 40586, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 8506, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 1200128, "loki_bytes_delta": 4431872, "errors_injected_500": 727, "slow_requests_gt500ms": 671, "traces_in_tempo": {"all_http": 3919, "error": 89, "slow_gt500ms": 80}, "kept_ratio": {"errors": 0.122, "slow": 0.119}}
{"profile": "head10", "post": {"requests": 39840, "ok": 39078, "errors": {"500": 762}, "seconds": 30, "ok_rps": 1302.6, "error_ratio": 0.0191, "p50_ms": 21.48, "p95_ms": 79.8, "p99_ms": 622.79}, "docker_stats": {"worker": {"avg_cpu_percent": 1.3, "peak_mem_mib": 60.2}, "api": {"avg_cpu_percent": 69.4, "peak_mem_mib": 74.0}, "otel-collector": {"avg_cpu_percent": 1.9, "peak_mem_mib": 68.9}, "tempo": {"avg_cpu_percent": 0.6, "peak_mem_mib": 755.1}, "lb": {"avg_cpu_percent": 19.9, "peak_mem_mib": 21.7}, "grafana": {"avg_cpu_percent": 1.5, "peak_mem_mib": 325.1}, "rabbitmq": {"avg_cpu_percent": 10.9, "peak_mem_mib": 194.5}, "postgres": {"avg_cpu_percent": 91.7, "peak_mem_mib": 303.6}, "loki": {"avg_cpu_percent": 4.6, "peak_mem_mib": 281.0}, "seaweedfs": {"avg_cpu_percent": 1.2, "peak_mem_mib": 494.4}, "prometheus": {"avg_cpu_percent": 0.7, "peak_mem_mib": 61.0}, "valkey": {"avg_cpu_percent": 5.0, "peak_mem_mib": 11.4}}, "collector_delta": {"receiver_accepted_spans": 9291, "receiver_accepted_log_records": 45766, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 9458, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 32768, "loki_bytes_delta": 32833536, "errors_injected_500": 774, "slow_requests_gt500ms": 802, "traces_in_tempo": {"all_http": 4411, "error": 81, "slow_gt500ms": 93}, "kept_ratio": {"errors": 0.105, "slow": 0.116}}
{"profile": "tail", "post": {"requests": 39275, "ok": 38469, "errors": {"500": 806}, "seconds": 30, "ok_rps": 1282.3, "error_ratio": 0.0205, "p50_ms": 23.74, "p95_ms": 86.59, "p99_ms": 631.13}, "docker_stats": {"worker": {"avg_cpu_percent": 2.1, "peak_mem_mib": 59.8}, "api": {"avg_cpu_percent": 77.1, "peak_mem_mib": 75.4}, "otel-collector": {"avg_cpu_percent": 4.6, "peak_mem_mib": 105.8}, "tempo": {"avg_cpu_percent": 0.8, "peak_mem_mib": 719.9}, "lb": {"avg_cpu_percent": 19.7, "peak_mem_mib": 22.0}, "grafana": {"avg_cpu_percent": 1.0, "peak_mem_mib": 325.2}, "rabbitmq": {"avg_cpu_percent": 12.5, "peak_mem_mib": 185.8}, "postgres": {"avg_cpu_percent": 93.0, "peak_mem_mib": 302.7}, "loki": {"avg_cpu_percent": 5.8, "peak_mem_mib": 312.2}, "seaweedfs": {"avg_cpu_percent": 1.4, "peak_mem_mib": 494.4}, "prometheus": {"avg_cpu_percent": 0.6, "peak_mem_mib": 60.9}, "valkey": {"avg_cpu_percent": 5.1, "peak_mem_mib": 11.4}}, "collector_delta": {"receiver_accepted_spans": 91490, "receiver_accepted_log_records": 45024, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 13023, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 401408, "loki_bytes_delta": 34607104, "errors_injected_500": 818, "slow_requests_gt500ms": 793, "traces_in_tempo": {"all_http": 5000, "error": 907, "slow_gt500ms": 883}, "kept_ratio": {"errors": 1.109, "slow": 1.113}}
{"profile": "tail", "post": {"requests": 39821, "ok": 38989, "errors": {"500": 832}, "seconds": 30, "ok_rps": 1299.6, "error_ratio": 0.0209, "p50_ms": 23.35, "p95_ms": 93.86, "p99_ms": 626.16}, "docker_stats": {"worker": {"avg_cpu_percent": 1.3, "peak_mem_mib": 64.2}, "api": {"avg_cpu_percent": 77.7, "peak_mem_mib": 75.5}, "otel-collector": {"avg_cpu_percent": 3.4, "peak_mem_mib": 108.4}, "tempo": {"avg_cpu_percent": 1.9, "peak_mem_mib": 443.0}, "lb": {"avg_cpu_percent": 18.6, "peak_mem_mib": 21.6}, "grafana": {"avg_cpu_percent": 0.7, "peak_mem_mib": 325.1}, "rabbitmq": {"avg_cpu_percent": 9.5, "peak_mem_mib": 175.5}, "postgres": {"avg_cpu_percent": 88.4, "peak_mem_mib": 303.5}, "loki": {"avg_cpu_percent": 4.3, "peak_mem_mib": 317.3}, "seaweedfs": {"avg_cpu_percent": 4.2, "peak_mem_mib": 692.3}, "prometheus": {"avg_cpu_percent": 0.5, "peak_mem_mib": 61.1}, "valkey": {"avg_cpu_percent": 4.9, "peak_mem_mib": 11.7}}, "collector_delta": {"receiver_accepted_spans": 92273, "receiver_accepted_log_records": 45427, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 13252, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 1703936, "loki_bytes_delta": 32804864, "errors_injected_500": 837, "slow_requests_gt500ms": 743, "traces_in_tempo": {"all_http": 5000, "error": 925, "slow_gt500ms": 834}, "kept_ratio": {"errors": 1.105, "slow": 1.122}}
saved
exit=0

$ python research/plot_obs_cost.py --out evidence\recheck_2026-11-12
saved
exit=0

$ python tests/smoke.py --base-url http://127.0.0.1:8080
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
exit=0

$ python -m unittest discover -s student-tests -p test_observability.py -v
test_logs_are_structured_and_linked_by_trace_id (test_observability.ObservabilityTest.test_logs_are_structured_and_linked_by_trace_id) ... ok
test_metrics_for_http_jobs_queue_pool_worker (test_observability.ObservabilityTest.test_metrics_for_http_jobs_queue_pool_worker) ... ok
test_no_high_cardinality_or_secret_labels (test_observability.ObservabilityTest.test_no_high_cardinality_or_secret_labels) ... ok
test_one_trace_spans_api_outbox_broker_and_worker (test_observability.ObservabilityTest.test_one_trace_spans_api_outbox_broker_and_worker) ... ok
test_secrets_and_message_bodies_are_not_exported (test_observability.ObservabilityTest.test_secrets_and_message_bodies_are_not_exported) ... ok

----------------------------------------------------------------------
Ran 5 tests in 12.547s

OK
exit=0
