forked from mtusi2026/mtusi2026lab6
238 lines
7.9 KiB
Python
238 lines
7.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Fire and resolve the two grade-4 Prometheus alerts."""
|
|
|
|
import argparse
|
|
import concurrent.futures
|
|
import json
|
|
import subprocess
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
|
|
from attachment_memory import cleanup
|
|
from client import Client
|
|
|
|
|
|
def compose(*args, check=True):
|
|
return subprocess.run(
|
|
["docker", "compose", *args],
|
|
check=check,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
|
|
|
|
def json_get(base_url, path, params=None, timeout=10):
|
|
query = "?" + urllib.parse.urlencode(params) if params else ""
|
|
with urllib.request.urlopen(base_url.rstrip("/") + path + query, timeout=timeout) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def wait_for(description, action, timeout, interval=2):
|
|
deadline = time.monotonic() + timeout
|
|
last_error = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
value = action()
|
|
if value:
|
|
return value
|
|
except (AssertionError, KeyError, OSError, urllib.error.URLError, ValueError) as error:
|
|
last_error = error
|
|
time.sleep(interval)
|
|
raise AssertionError(f"timeout waiting for {description}: {last_error}")
|
|
|
|
|
|
def alert(prometheus_url, alert_name, state=None):
|
|
payload = json_get(prometheus_url, "/api/v1/alerts")
|
|
assert payload["status"] == "success", payload
|
|
for item in payload["data"]["alerts"]:
|
|
if item.get("labels", {}).get("alertname") != alert_name:
|
|
continue
|
|
if state is None or item.get("state") == state:
|
|
return item
|
|
return None
|
|
|
|
|
|
def alert_resolved(prometheus_url, alert_name):
|
|
return alert(prometheus_url, alert_name) is None
|
|
|
|
|
|
def prometheus_scalar(base_url, expression):
|
|
payload = json_get(base_url, "/api/v1/query", {"query": expression})
|
|
results = payload["data"]["result"]
|
|
return float(results[0]["value"][1]) if results else 0.0
|
|
|
|
|
|
def trace_id_for_request(loki_url, request_id):
|
|
payload = json_get(
|
|
loki_url,
|
|
"/loki/api/v1/query_range",
|
|
{
|
|
"query": f'{{service_name="messenger-api"}} | request_id="{request_id}"',
|
|
"start": str(time.time_ns() - 10 * 60 * 1_000_000_000),
|
|
"end": str(time.time_ns()),
|
|
"direction": "backward",
|
|
"limit": "20",
|
|
},
|
|
)
|
|
for result in payload["data"]["result"]:
|
|
trace_id = result.get("stream", {}).get("trace_id")
|
|
if trace_id and len(trace_id) == 32:
|
|
return trace_id
|
|
return None
|
|
|
|
|
|
def db_error_span(tempo_url, trace_id):
|
|
payload = json_get(tempo_url, f"/api/traces/{trace_id}")
|
|
for batch in payload.get("batches", []):
|
|
for scope in batch.get("scopeSpans", []):
|
|
for span in scope.get("spans", []):
|
|
if (
|
|
span.get("name", "").startswith("postgresql.")
|
|
and span.get("status", {}).get("code") == "STATUS_CODE_ERROR"
|
|
):
|
|
return span.get("name")
|
|
return None
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--base-url", default="http://localhost:8080")
|
|
parser.add_argument("--prometheus-url", default="http://localhost:9090")
|
|
parser.add_argument("--loki-url", default="http://localhost:3100")
|
|
parser.add_argument("--tempo-url", default="http://localhost:3200")
|
|
parser.add_argument("--timeout", type=int, default=150)
|
|
args = parser.parse_args()
|
|
|
|
client = Client(args.base_url, timeout=15)
|
|
alice, bob, eve, chat = client.bootstrap()
|
|
users = [alice["id"], bob["id"], eve["id"]]
|
|
postgres_paused = False
|
|
queue_result = {}
|
|
database_result = {}
|
|
try:
|
|
compose("stop", "worker")
|
|
queue_started = time.monotonic()
|
|
content = open("tests/fixtures/sample.png", "rb").read()
|
|
response = client.call(
|
|
"POST",
|
|
f'/api/v1/chats/{chat["id"]}/attachments?filename=queue-alert.png',
|
|
202,
|
|
actor=alice,
|
|
raw=content,
|
|
headers={"Content-Type": "image/png"},
|
|
schema="Attachment",
|
|
)
|
|
queue_alert = wait_for(
|
|
"queue lag alert firing",
|
|
lambda: alert(
|
|
args.prometheus_url, "MessengerQueueLagGrowing", "firing"
|
|
),
|
|
args.timeout,
|
|
)
|
|
queue_result = {
|
|
"firing_after_seconds": round(time.monotonic() - queue_started, 3),
|
|
"depth": prometheus_scalar(args.prometheus_url, "max(messenger_queue_depth)"),
|
|
"oldest_age_seconds": prometheus_scalar(
|
|
args.prometheus_url, "max(messenger_queue_oldest_job_age_seconds)"
|
|
),
|
|
"state": queue_alert["state"],
|
|
}
|
|
compose("up", "-d", "worker")
|
|
client.wait_ready(alice, response["json"], args.timeout)
|
|
queue_recovery_started = time.monotonic()
|
|
wait_for(
|
|
"queue lag alert resolved",
|
|
lambda: alert_resolved(args.prometheus_url, "MessengerQueueLagGrowing"),
|
|
args.timeout,
|
|
)
|
|
queue_result["resolved_after_seconds"] = round(
|
|
time.monotonic() - queue_recovery_started, 3
|
|
)
|
|
|
|
compose("pause", "postgres")
|
|
postgres_paused = True
|
|
database_started = time.monotonic()
|
|
request_ids = [str(uuid.uuid4()) for _ in range(8)]
|
|
|
|
def failed_request(request_id):
|
|
isolated_client = Client(args.base_url, timeout=15)
|
|
isolated_client.call(
|
|
"GET",
|
|
"/api/v1/chats",
|
|
503,
|
|
actor=alice,
|
|
headers={"X-Request-Id": request_id},
|
|
)
|
|
return request_id
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
|
list(pool.map(failed_request, request_ids))
|
|
db_alert = wait_for(
|
|
"API high error rate alert firing",
|
|
lambda: alert(
|
|
args.prometheus_url, "MessengerAPIHighErrorRate", "firing"
|
|
),
|
|
args.timeout,
|
|
)
|
|
database_result = {
|
|
"firing_after_seconds": round(time.monotonic() - database_started, 3),
|
|
"failed_requests": len(request_ids),
|
|
"state": db_alert["state"],
|
|
}
|
|
compose("unpause", "postgres")
|
|
postgres_paused = False
|
|
wait_for(
|
|
"API readiness after PostgreSQL recovery",
|
|
lambda: client.call("GET", "/health/ready", schema="Health"),
|
|
30,
|
|
)
|
|
trace_id = wait_for(
|
|
"database failure correlated log",
|
|
lambda: trace_id_for_request(args.loki_url, request_ids[0]),
|
|
args.timeout,
|
|
)
|
|
error_span = wait_for(
|
|
"database error span",
|
|
lambda: db_error_span(args.tempo_url, trace_id),
|
|
args.timeout,
|
|
)
|
|
database_result.update({"trace_id": trace_id, "error_span": error_span})
|
|
database_recovery_started = time.monotonic()
|
|
wait_for(
|
|
"API high error rate alert resolved",
|
|
lambda: alert_resolved(args.prometheus_url, "MessengerAPIHighErrorRate"),
|
|
args.timeout,
|
|
)
|
|
database_result["resolved_after_seconds"] = round(
|
|
time.monotonic() - database_recovery_started, 3
|
|
)
|
|
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "PASS",
|
|
"queue_lag": queue_result,
|
|
"database_outage": database_result,
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
finally:
|
|
if postgres_paused:
|
|
compose("unpause", "postgres", check=False)
|
|
compose("up", "-d", "worker", check=False)
|
|
wait_for(
|
|
"PostgreSQL before cleanup",
|
|
lambda: compose("exec", "-T", "postgres", "pg_isready", "-U", "messenger").returncode == 0,
|
|
30,
|
|
)
|
|
cleanup(chat["id"], users)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|