Files
mtusi2026lab6/tests/observability_alerts.py

238 lines
7.9 KiB
Python

#!/usr/bin/env python3
"""Fire and resolve the two grade-4 Prometheus alerts."""
import argparse
import concurrent.futures
import json
import subprocess
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from attachment_memory import cleanup
from client import Client
def compose(*args, check=True):
return subprocess.run(
["docker", "compose", *args],
check=check,
text=True,
capture_output=True,
)
def json_get(base_url, path, params=None, timeout=10):
query = "?" + urllib.parse.urlencode(params) if params else ""
with urllib.request.urlopen(base_url.rstrip("/") + path + query, timeout=timeout) as response:
return json.load(response)
def wait_for(description, action, timeout, interval=2):
deadline = time.monotonic() + timeout
last_error = None
while time.monotonic() < deadline:
try:
value = action()
if value:
return value
except (AssertionError, KeyError, OSError, urllib.error.URLError, ValueError) as error:
last_error = error
time.sleep(interval)
raise AssertionError(f"timeout waiting for {description}: {last_error}")
def alert(prometheus_url, alert_name, state=None):
payload = json_get(prometheus_url, "/api/v1/alerts")
assert payload["status"] == "success", payload
for item in payload["data"]["alerts"]:
if item.get("labels", {}).get("alertname") != alert_name:
continue
if state is None or item.get("state") == state:
return item
return None
def alert_resolved(prometheus_url, alert_name):
return alert(prometheus_url, alert_name) is None
def prometheus_scalar(base_url, expression):
payload = json_get(base_url, "/api/v1/query", {"query": expression})
results = payload["data"]["result"]
return float(results[0]["value"][1]) if results else 0.0
def trace_id_for_request(loki_url, request_id):
payload = json_get(
loki_url,
"/loki/api/v1/query_range",
{
"query": f'{{service_name="messenger-api"}} | request_id="{request_id}"',
"start": str(time.time_ns() - 10 * 60 * 1_000_000_000),
"end": str(time.time_ns()),
"direction": "backward",
"limit": "20",
},
)
for result in payload["data"]["result"]:
trace_id = result.get("stream", {}).get("trace_id")
if trace_id and len(trace_id) == 32:
return trace_id
return None
def db_error_span(tempo_url, trace_id):
payload = json_get(tempo_url, f"/api/traces/{trace_id}")
for batch in payload.get("batches", []):
for scope in batch.get("scopeSpans", []):
for span in scope.get("spans", []):
if (
span.get("name", "").startswith("postgresql.")
and span.get("status", {}).get("code") == "STATUS_CODE_ERROR"
):
return span.get("name")
return None
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--prometheus-url", default="http://localhost:9090")
parser.add_argument("--loki-url", default="http://localhost:3100")
parser.add_argument("--tempo-url", default="http://localhost:3200")
parser.add_argument("--timeout", type=int, default=150)
args = parser.parse_args()
client = Client(args.base_url, timeout=15)
alice, bob, eve, chat = client.bootstrap()
users = [alice["id"], bob["id"], eve["id"]]
postgres_paused = False
queue_result = {}
database_result = {}
try:
compose("stop", "worker")
queue_started = time.monotonic()
content = open("tests/fixtures/sample.png", "rb").read()
response = client.call(
"POST",
f'/api/v1/chats/{chat["id"]}/attachments?filename=queue-alert.png',
202,
actor=alice,
raw=content,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
queue_alert = wait_for(
"queue lag alert firing",
lambda: alert(
args.prometheus_url, "MessengerQueueLagGrowing", "firing"
),
args.timeout,
)
queue_result = {
"firing_after_seconds": round(time.monotonic() - queue_started, 3),
"depth": prometheus_scalar(args.prometheus_url, "max(messenger_queue_depth)"),
"oldest_age_seconds": prometheus_scalar(
args.prometheus_url, "max(messenger_queue_oldest_job_age_seconds)"
),
"state": queue_alert["state"],
}
compose("up", "-d", "worker")
client.wait_ready(alice, response["json"], args.timeout)
queue_recovery_started = time.monotonic()
wait_for(
"queue lag alert resolved",
lambda: alert_resolved(args.prometheus_url, "MessengerQueueLagGrowing"),
args.timeout,
)
queue_result["resolved_after_seconds"] = round(
time.monotonic() - queue_recovery_started, 3
)
compose("pause", "postgres")
postgres_paused = True
database_started = time.monotonic()
request_ids = [str(uuid.uuid4()) for _ in range(8)]
def failed_request(request_id):
isolated_client = Client(args.base_url, timeout=15)
isolated_client.call(
"GET",
"/api/v1/chats",
503,
actor=alice,
headers={"X-Request-Id": request_id},
)
return request_id
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
list(pool.map(failed_request, request_ids))
db_alert = wait_for(
"API high error rate alert firing",
lambda: alert(
args.prometheus_url, "MessengerAPIHighErrorRate", "firing"
),
args.timeout,
)
database_result = {
"firing_after_seconds": round(time.monotonic() - database_started, 3),
"failed_requests": len(request_ids),
"state": db_alert["state"],
}
compose("unpause", "postgres")
postgres_paused = False
wait_for(
"API readiness after PostgreSQL recovery",
lambda: client.call("GET", "/health/ready", schema="Health"),
30,
)
trace_id = wait_for(
"database failure correlated log",
lambda: trace_id_for_request(args.loki_url, request_ids[0]),
args.timeout,
)
error_span = wait_for(
"database error span",
lambda: db_error_span(args.tempo_url, trace_id),
args.timeout,
)
database_result.update({"trace_id": trace_id, "error_span": error_span})
database_recovery_started = time.monotonic()
wait_for(
"API high error rate alert resolved",
lambda: alert_resolved(args.prometheus_url, "MessengerAPIHighErrorRate"),
args.timeout,
)
database_result["resolved_after_seconds"] = round(
time.monotonic() - database_recovery_started, 3
)
print(
json.dumps(
{
"status": "PASS",
"queue_lag": queue_result,
"database_outage": database_result,
},
ensure_ascii=False,
indent=2,
)
)
finally:
if postgres_paused:
compose("unpause", "postgres", check=False)
compose("up", "-d", "worker", check=False)
wait_for(
"PostgreSQL before cleanup",
lambda: compose("exec", "-T", "postgres", "pg_isready", "-U", "messenger").returncode == 0,
30,
)
cleanup(chat["id"], users)
if __name__ == "__main__":
main()