date: 2026-11-06T18:12:24Z  commit: cd9d91550aa4eeba31489e07b71ee2767acfc7d8
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ python3 scripts/obs_incidents.py database
{
  "started": "2026-11-06T18:12:25+00:00",
  "finished": "2026-11-06T18:14:15+00:00",
  "result": "PASS",
  "scenario": "database",
  "observations": {
    "paused": "2026-11-06T18:12:37+00:00",
    "restored": "2026-11-06T18:13:13+00:00",
    "requests": [
      {
        "request_id": "db-incident-8985ff30724f-10",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-11",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-12",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-13",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-14",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-15",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-16",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-17",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-18",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-19",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-20",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-21",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-22",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-23",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-24",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-25",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-26",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-27",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-28",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-29",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-30",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-31",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-32",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-33",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-34",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-35",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-36",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-37",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-38",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-39",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-40",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-41",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-42",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-43",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-44",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-45",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-46",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-47",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-48",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-49",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-50",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-51",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-52",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-53",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-54",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-55",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-56",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-57",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-58",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-59",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-60",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-61",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-62",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-63",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-64",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-65",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-66",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-67",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-68",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-69",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-70",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-71",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-72",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-73",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-74",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-75",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-76",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-77",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-78",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-79",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-80",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-81",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-82",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-83",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-84",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-85",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-86",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-87",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-88",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-89",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-90",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-91",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-92",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-93",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-94",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-95",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-96",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-97",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-98",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-99",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-100",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-101",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-102",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-103",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-104",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-105",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-106",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-107",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-108",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-109",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-110",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-111",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-112",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-113",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-114",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-115",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-116",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-117",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-118",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-119",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-120",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-121",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-122",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-123",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-124",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-125",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-126",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-127",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-128",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-129",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-130",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-131",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-132",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-133",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-134",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-135",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-136",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-137",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-138",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-139",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-140",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-141",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-142",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-143",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-144",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-145",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-146",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-147",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-148",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-149",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-150",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-151",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-152",
        "status": 503
      },
      {
        "request_id": "db-incident-8985ff30724f-153",
        "status": 503
      }
    ],
    "signals": {
      "error_ratio": 1.0,
      "pool_waiting": 0.0
    },
    "firing": {
      "utc": "2026-11-06T18:13:13+00:00",
      "state": "firing",
      "alerts": [
        {
          "labels": {
            "alertname": "MessengerHighErrorRate",
            "severity": "warning"
          },
          "annotations": {
            "description": "Availability target is 99% over 1 hour; health and client 4xx are excluded from failures.",
            "runbook": "infra/prometheus/runbooks.md#messengerhigherrorrate",
            "summary": "More than 5% business HTTP 5xx over 1 minute"
          },
          "state": "firing",
          "activeAt": "2026-11-06T18:12:58.389340978Z",
          "value": "1e+00"
        }
      ]
    },
    "resolved": {
      "utc": "2026-11-06T18:14:14+00:00",
      "state": "resolved",
      "alerts": []
    },
    "dependency_traces": [
      {
        "trace_id": "56a65d4cf3881f888a2550ca344f578e",
        "db_errors": [
          {
            "name": "SELECT",
            "kind": "SPAN_KIND_CLIENT",
            "status": "STATUS_CODE_ERROR",
            "duration_ms": 2002.331
          }
        ]
      },
      {
        "trace_id": "36ab8690d98f80687f5983bfa9a50d3c",
        "db_errors": [
          {
            "name": "SELECT",
            "kind": "SPAN_KIND_CLIENT",
            "status": "STATUS_CODE_ERROR",
            "duration_ms": 2004.439
          }
        ]
      },
      {
        "trace_id": "ef191d8d6bc7ab4d4d87feefaa464ffe",
        "db_errors": [
          {
            "name": "SELECT",
            "kind": "SPAN_KIND_CLIENT",
            "status": "STATUS_CODE_ERROR",
            "duration_ms": 2010.771
          }
        ]
      }
    ]
  }
}
exit_code: 0  elapsed_seconds: 110.856
