Files
mtusi2026lab6/scripts/slo_watch.py
T

70 lines
2.7 KiB
Python

#!/usr/bin/env python3
"""Capture local Prometheus alert states and SLI values without credentials."""
import argparse
import json
import time
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urlencode
from urllib.request import urlopen
QUERIES = {
"http_5xx_ratio_30s": 'sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])),0.001)',
"queue_oldest_seconds": 'max(messenger_queue_oldest_age_seconds{queue_name="attachments.process"})',
"ready_slow_ratio_30s": '(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])) - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[30s]))) / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])),0.001)',
"collector_failed_spans": 'sum(otelcol_exporter_send_failed_spans)',
}
def fetch(base, path, params=None):
url = base + path + ("?" + urlencode(params) if params else "")
with urlopen(url, timeout=4) as response:
result = json.load(response)
if result["status"] != "success":
raise RuntimeError(result)
return result["data"]
def snapshot(base):
groups = fetch(base, "/api/v1/rules", {"type": "alert"})["groups"]
alerts = [rule for group in groups for rule in group["rules"]]
row = {
"timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"alerts": {item["name"]: item["state"]
for item in alerts if item["name"].startswith("Lab6")},
"values": {},
}
for name, query in QUERIES.items():
values = fetch(base, "/api/v1/query", {"query": query})["result"]
row["values"][name] = float(values[0]["value"][1]) if values else None
return row
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--prometheus", default="http://127.0.0.1:9090")
parser.add_argument("--duration", type=int, default=60)
parser.add_argument("--interval", type=int, default=5)
parser.add_argument("--output", required=True)
args = parser.parse_args()
if not 5 <= args.duration <= 900 or not 1 <= args.interval <= 30:
parser.error("invalid duration or interval")
path = Path(args.output)
path.parent.mkdir(parents=True, exist_ok=True)
end = time.monotonic() + args.duration
with path.open("w", encoding="utf-8") as out:
while True:
row = snapshot(args.prometheus)
out.write(json.dumps(row, sort_keys=True) + "\n")
out.flush()
if time.monotonic() + args.interval > end:
break
time.sleep(args.interval)
print(f"saved {path}")
if __name__ == "__main__":
main()