date: 2026-11-06T23:48:04Z  commit: bc1d7aa7713858acb5d65b61d07e69a640f1ee97
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_incidents.py database
{
  "started": "2026-11-06T23:48:04+00:00",
  "finished": "2026-11-06T23:49:55+00:00",
  "result": "PASS",
  "scenario": "database",
  "observations": {
    "paused": "2026-11-06T23:48:16+00:00",
    "restored": "2026-11-06T23:48:59+00:00",
    "requests": [
      {
        "request_id": "db-incident-8b5642602908-10",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-11",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-12",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-13",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-14",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-15",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-16",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-17",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-18",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-19",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-20",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-21",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-22",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-23",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-24",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-25",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-26",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-27",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-28",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-29",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-30",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-31",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-32",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-33",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-34",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-35",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-36",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-37",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-38",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-39",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-40",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-41",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-42",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-43",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-44",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-45",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-46",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-47",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-48",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-49",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-50",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-51",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-52",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-53",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-54",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-55",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-56",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-57",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-58",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-59",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-60",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-61",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-62",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-63",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-64",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-65",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-66",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-67",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-68",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-69",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-70",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-71",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-72",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-73",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-74",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-75",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-76",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-77",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-78",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-79",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-80",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-81",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-82",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-83",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-84",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-85",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-86",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-87",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-88",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-89",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-90",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-91",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-92",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-93",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-94",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-95",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-96",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-97",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-98",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-99",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-100",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-101",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-102",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-103",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-104",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-105",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-106",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-107",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-108",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-109",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-110",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-111",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-112",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-113",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-114",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-115",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-116",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-117",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-118",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-119",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-120",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-121",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-122",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-123",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-124",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-125",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-126",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-127",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-128",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-129",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-130",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-131",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-132",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-133",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-134",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-135",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-136",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-137",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-138",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-139",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-140",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-141",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-142",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-143",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-144",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-145",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-146",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-147",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-148",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-149",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-150",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-151",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-152",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-153",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-154",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-155",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-156",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-157",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-158",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-159",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-160",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-161",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-162",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-163",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-164",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-165",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-166",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-167",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-168",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-169",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-170",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-171",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-172",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-173",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-174",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-175",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-176",
        "status": 503
      },
      {
        "request_id": "db-incident-8b5642602908-177",
        "status": 503
      }
    ],
    "signals": {
      "error_ratio": 0.9181136211080942,
      "pool_waiting": 0.0
    },
    "firing": {
      "utc": "2026-11-06T23:48:59+00:00",
      "state": "firing",
      "alerts": [
        {
          "labels": {
            "alertname": "MessengerHighErrorRate",
            "severity": "warning"
          },
          "annotations": {
            "description": "Availability target is 99% over 1 hour; health and client 4xx are excluded from failures.",
            "runbook": "infra/prometheus/runbooks.md#messengerhigherrorrate",
            "summary": "More than 5% business HTTP 5xx over 1 minute"
          },
          "state": "firing",
          "activeAt": "2026-11-06T23:48:43.389340978Z",
          "value": "9.181136211080942e-01"
        }
      ]
    },
    "resolved": {
      "utc": "2026-11-06T23:49:54+00:00",
      "state": "resolved",
      "alerts": []
    },
    "dependency_traces": [
      {
        "trace_id": "1cee5df1144b8c4eb2b801d449bfc0d4",
        "db_errors": [
          {
            "name": "SELECT",
            "kind": "SPAN_KIND_CLIENT",
            "status": "STATUS_CODE_ERROR",
            "duration_ms": 2002.866
          }
        ]
      }
    ]
  }
}
exit_code: 0  elapsed_seconds: 111.140
