date: 2026-11-06T18:14:15Z  commit: cd9d91550aa4eeba31489e07b71ee2767acfc7d8
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_incidents.py storage
{
  "started": "2026-11-06T18:14:15+00:00",
  "finished": "2026-11-06T18:16:39+00:00",
  "result": "PASS",
  "scenario": "storage",
  "observations": {
    "stopped": "2026-11-06T18:14:29+00:00",
    "restored": "2026-11-06T18:16:16+00:00",
    "peak_memory_percent_of_limit": 98.7,
    "smoke": "python3 tests/smoke.py --base-url \"http://127.0.0.1:18086\" \nPASS: lab6 public API scenario; infrastructure and grading evidence remain separate.\n",
    "successful_probes": 74,
    "stored_probe_traces": 72,
    "lost_probe_traces": 2,
    "memory_samples": [
      {
        "utc": "2026-11-06T18:14:16+00:00",
        "containers": [
          {
            "Name": "mtusi2026lab6-api-1",
            "MemUsage": "246.6MiB / 512MiB",
            "MemPerc": "48.16%",
            "CPUPerc": "1.94%"
          },
          {
            "Name": "mtusi2026lab6-api-2",
            "MemUsage": "241.7MiB / 512MiB",
            "MemPerc": "47.20%",
            "CPUPerc": "1.12%"
          },
          {
            "Name": "mtusi2026lab6-otel-collector-1",
            "MemUsage": "246.2MiB / 256MiB",
            "MemPerc": "96.18%",
            "CPUPerc": "0.63%"
          },
          {
            "Name": "mtusi2026lab6-relay-1",
            "MemUsage": "119.4MiB / 192MiB",
            "MemPerc": "62.18%",
            "CPUPerc": "0.38%"
          },
          {
            "Name": "mtusi2026lab6-worker-1",
            "MemUsage": "121.3MiB / 1GiB",
            "MemPerc": "11.84%",
            "CPUPerc": "0.10%"
          }
        ]
      },
      {
        "utc": "2026-11-06T18:14:45+00:00",
        "containers": [
          {
            "Name": "mtusi2026lab6-api-1",
            "MemUsage": "259.7MiB / 512MiB",
            "MemPerc": "50.72%",
            "CPUPerc": "1.59%"
          },
          {
            "Name": "mtusi2026lab6-api-2",
            "MemUsage": "249.4MiB / 512MiB",
            "MemPerc": "48.71%",
            "CPUPerc": "1.56%"
          },
          {
            "Name": "mtusi2026lab6-otel-collector-1",
            "MemUsage": "248.5MiB / 256MiB",
            "MemPerc": "97.08%",
            "CPUPerc": "0.30%"
          },
          {
            "Name": "mtusi2026lab6-relay-1",
            "MemUsage": "120.2MiB / 192MiB",
            "MemPerc": "62.59%",
            "CPUPerc": "0.48%"
          },
          {
            "Name": "mtusi2026lab6-worker-1",
            "MemUsage": "253.4MiB / 1GiB",
            "MemPerc": "24.75%",
            "CPUPerc": "0.35%"
          }
        ]
      },
      {
        "utc": "2026-11-06T18:15:03+00:00",
        "containers": [
          {
            "Name": "mtusi2026lab6-api-1",
            "MemUsage": "259.5MiB / 512MiB",
            "MemPerc": "50.69%",
            "CPUPerc": "1.10%"
          },
          {
            "Name": "mtusi2026lab6-api-2",
            "MemUsage": "249.4MiB / 512MiB",
            "MemPerc": "48.71%",
            "CPUPerc": "1.19%"
          },
          {
            "Name": "mtusi2026lab6-otel-collector-1",
            "MemUsage": "250.5MiB / 256MiB",
            "MemPerc": "97.87%",
            "CPUPerc": "0.20%"
          },
          {
            "Name": "mtusi2026lab6-relay-1",
            "MemUsage": "119.9MiB / 192MiB",
            "MemPerc": "62.46%",
            "CPUPerc": "0.18%"
          },
          {
            "Name": "mtusi2026lab6-worker-1",
            "MemUsage": "253.5MiB / 1GiB",
            "MemPerc": "24.75%",
            "CPUPerc": "2.07%"
          }
        ]
      },
      {
        "utc": "2026-11-06T18:15:21+00:00",
        "containers": [
          {
            "Name": "mtusi2026lab6-api-1",
            "MemUsage": "259.6MiB / 512MiB",
            "MemPerc": "50.70%",
            "CPUPerc": "2.63%"
          },
          {
            "Name": "mtusi2026lab6-api-2",
            "MemUsage": "249.5MiB / 512MiB",
            "MemPerc": "48.73%",
            "CPUPerc": "1.35%"
          },
          {
            "Name": "mtusi2026lab6-otel-collector-1",
            "MemUsage": "250.6MiB / 256MiB",
            "MemPerc": "97.89%",
            "CPUPerc": "1.18%"
          },
          {
            "Name": "mtusi2026lab6-relay-1",
            "MemUsage": "119.9MiB / 192MiB",
            "MemPerc": "62.47%",
            "CPUPerc": "1.77%"
          },
          {
            "Name": "mtusi2026lab6-worker-1",
            "MemUsage": "253.4MiB / 1GiB",
            "MemPerc": "24.75%",
            "CPUPerc": "0.75%"
          }
        ]
      },
      {
        "utc": "2026-11-06T18:15:39+00:00",
        "containers": [
          {
            "Name": "mtusi2026lab6-api-1",
            "MemUsage": "259.6MiB / 512MiB",
            "MemPerc": "50.71%",
            "CPUPerc": "0.50%"
          },
          {
            "Name": "mtusi2026lab6-api-2",
            "MemUsage": "249.1MiB / 512MiB",
            "MemPerc": "48.66%",
            "CPUPerc": "0.47%"
          },
          {
            "Name": "mtusi2026lab6-otel-collector-1",
            "MemUsage": "250.6MiB / 256MiB",
            "MemPerc": "97.88%",
            "CPUPerc": "0.07%"
          },
          {
            "Name": "mtusi2026lab6-relay-1",
            "MemUsage": "120MiB / 192MiB",
            "MemPerc": "62.48%",
            "CPUPerc": "0.18%"
          },
          {
            "Name": "mtusi2026lab6-worker-1",
            "MemUsage": "253.4MiB / 1GiB",
            "MemPerc": "24.75%",
            "CPUPerc": "0.20%"
          }
        ]
      },
      {
        "utc": "2026-11-06T18:16:36+00:00",
        "containers": [
          {
            "Name": "mtusi2026lab6-api-1",
            "MemUsage": "259.6MiB / 512MiB",
            "MemPerc": "50.70%",
            "CPUPerc": "1.35%"
          },
          {
            "Name": "mtusi2026lab6-api-2",
            "MemUsage": "249.1MiB / 512MiB",
            "MemPerc": "48.65%",
            "CPUPerc": "1.59%"
          },
          {
            "Name": "mtusi2026lab6-otel-collector-1",
            "MemUsage": "252.7MiB / 256MiB",
            "MemPerc": "98.70%",
            "CPUPerc": "0.23%"
          },
          {
            "Name": "mtusi2026lab6-relay-1",
            "MemUsage": "120MiB / 192MiB",
            "MemPerc": "62.49%",
            "CPUPerc": "0.77%"
          },
          {
            "Name": "mtusi2026lab6-worker-1",
            "MemUsage": "253.4MiB / 1GiB",
            "MemPerc": "24.75%",
            "CPUPerc": "0.29%"
          }
        ]
      }
    ],
    "collector_before": {
      "otelcol_exporter_queue_capacity{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 20000.0,
      "otelcol_exporter_queue_capacity{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 50000.0,
      "otelcol_exporter_queue_capacity{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 20000.0,
      "otelcol_exporter_queue_size{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 0.0,
      "otelcol_exporter_queue_size{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 0.0,
      "otelcol_exporter_queue_size{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 0.0,
      "otelcol_exporter_sent_log_records_total{exporter=\"otlp_http/loki\",server_address=\"loki\",server_port=\"3100\",url_path=\"/otlp/v1/logs\"}": 807.0,
      "otelcol_exporter_sent_metric_points_total{exporter=\"otlp_http/prometheus\",server_address=\"prometheus\",server_port=\"9090\",url_path=\"/api/v1/otlp/v1/metrics\"}": 656.0,
      "otelcol_exporter_sent_spans_total{exporter=\"otlp_http/tempo\",server_address=\"tempo\",server_port=\"4318\",url_path=\"/v1/traces\"}": 784.0,
      "otelcol_receiver_accepted_log_records_total{receiver=\"otlp\",transport=\"http\"}": 810.0,
      "otelcol_receiver_accepted_metric_points_total{receiver=\"otlp\",transport=\"http\"}": 656.0,
      "otelcol_receiver_accepted_spans_total{receiver=\"otlp\",transport=\"http\"}": 784.0
    },
    "collector_during": {
      "otelcol_exporter_queue_capacity{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 20000.0,
      "otelcol_exporter_queue_capacity{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 50000.0,
      "otelcol_exporter_queue_capacity{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 20000.0,
      "otelcol_exporter_queue_size{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 381.0,
      "otelcol_exporter_queue_size{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 1287.0,
      "otelcol_exporter_queue_size{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 465.0,
      "otelcol_exporter_send_failed_log_records_total{exporter=\"otlp_http/loki\",server_address=\"loki\",server_port=\"3100\",url_path=\"/otlp/v1/logs\"}": 11.0,
      "otelcol_exporter_send_failed_metric_points_total{exporter=\"otlp_http/prometheus\",server_address=\"prometheus\",server_port=\"9090\",url_path=\"/api/v1/otlp/v1/metrics\"}": 72.0,
      "otelcol_exporter_send_failed_spans_total{exporter=\"otlp_http/tempo\",server_address=\"tempo\",server_port=\"4318\",url_path=\"/v1/traces\"}": 440.0,
      "otelcol_exporter_sent_log_records_total{exporter=\"otlp_http/loki\",server_address=\"loki\",server_port=\"3100\",url_path=\"/otlp/v1/logs\"}": 833.0,
      "otelcol_exporter_sent_metric_points_total{exporter=\"otlp_http/prometheus\",server_address=\"prometheus\",server_port=\"9090\",url_path=\"/api/v1/otlp/v1/metrics\"}": 656.0,
      "otelcol_exporter_sent_spans_total{exporter=\"otlp_http/tempo\",server_address=\"tempo\",server_port=\"4318\",url_path=\"/v1/traces\"}": 807.0,
      "otelcol_receiver_accepted_log_records_total{receiver=\"otlp\",transport=\"http\"}": 1228.0,
      "otelcol_receiver_accepted_metric_points_total{receiver=\"otlp\",transport=\"http\"}": 2015.0,
      "otelcol_receiver_accepted_spans_total{receiver=\"otlp\",transport=\"http\"}": 1712.0
    },
    "collector_after": {
      "otelcol_exporter_queue_capacity{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 20000.0,
      "otelcol_exporter_queue_capacity{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 50000.0,
      "otelcol_exporter_queue_capacity{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 20000.0,
      "otelcol_exporter_queue_size{data_type=\"logs\",exporter=\"otlp_http/loki\"}": 0.0,
      "otelcol_exporter_queue_size{data_type=\"metrics\",exporter=\"otlp_http/prometheus\"}": 0.0,
      "otelcol_exporter_queue_size{data_type=\"traces\",exporter=\"otlp_http/tempo\"}": 0.0,
      "otelcol_exporter_send_failed_log_records_total{exporter=\"otlp_http/loki\",server_address=\"loki\",server_port=\"3100\",url_path=\"/otlp/v1/logs\"}": 11.0,
      "otelcol_exporter_send_failed_metric_points_total{exporter=\"otlp_http/prometheus\",server_address=\"prometheus\",server_port=\"9090\",url_path=\"/api/v1/otlp/v1/metrics\"}": 72.0,
      "otelcol_exporter_send_failed_spans_total{exporter=\"otlp_http/tempo\",server_address=\"tempo\",server_port=\"4318\",url_path=\"/v1/traces\"}": 440.0,
      "otelcol_exporter_sent_log_records_total{exporter=\"otlp_http/loki\",server_address=\"loki\",server_port=\"3100\",url_path=\"/otlp/v1/logs\"}": 1320.0,
      "otelcol_exporter_sent_metric_points_total{exporter=\"otlp_http/prometheus\",server_address=\"prometheus\",server_port=\"9090\",url_path=\"/api/v1/otlp/v1/metrics\"}": 2515.0,
      "otelcol_exporter_sent_spans_total{exporter=\"otlp_http/tempo\",server_address=\"tempo\",server_port=\"4318\",url_path=\"/v1/traces\"}": 1293.0,
      "otelcol_receiver_accepted_log_records_total{receiver=\"otlp\",transport=\"http\"}": 1335.0,
      "otelcol_receiver_accepted_metric_points_total{receiver=\"otlp\",transport=\"http\"}": 2587.0,
      "otelcol_receiver_accepted_spans_total{receiver=\"otlp\",transport=\"http\"}": 1733.0
    }
  }
}
exit_code: 0  elapsed_seconds: 143.965
