70 lines
2.7 KiB
Python
70 lines
2.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Capture local Prometheus alert states and SLI values without credentials."""
|
|
|
|
import argparse
|
|
import json
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from urllib.parse import urlencode
|
|
from urllib.request import urlopen
|
|
|
|
|
|
QUERIES = {
|
|
"http_5xx_ratio_30s": 'sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])),0.001)',
|
|
"queue_oldest_seconds": 'max(messenger_queue_oldest_age_seconds{queue_name="attachments.process"})',
|
|
"ready_slow_ratio_30s": '(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])) - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[30s]))) / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])),0.001)',
|
|
"collector_failed_spans": 'sum(otelcol_exporter_send_failed_spans)',
|
|
}
|
|
|
|
|
|
def fetch(base, path, params=None):
|
|
url = base + path + ("?" + urlencode(params) if params else "")
|
|
with urlopen(url, timeout=4) as response:
|
|
result = json.load(response)
|
|
if result["status"] != "success":
|
|
raise RuntimeError(result)
|
|
return result["data"]
|
|
|
|
|
|
def snapshot(base):
|
|
groups = fetch(base, "/api/v1/rules", {"type": "alert"})["groups"]
|
|
alerts = [rule for group in groups for rule in group["rules"]]
|
|
row = {
|
|
"timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
|
"alerts": {item["name"]: item["state"]
|
|
for item in alerts if item["name"].startswith("Lab6")},
|
|
"values": {},
|
|
}
|
|
for name, query in QUERIES.items():
|
|
values = fetch(base, "/api/v1/query", {"query": query})["result"]
|
|
row["values"][name] = float(values[0]["value"][1]) if values else None
|
|
return row
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--prometheus", default="http://127.0.0.1:9090")
|
|
parser.add_argument("--duration", type=int, default=60)
|
|
parser.add_argument("--interval", type=int, default=5)
|
|
parser.add_argument("--output", required=True)
|
|
args = parser.parse_args()
|
|
if not 5 <= args.duration <= 900 or not 1 <= args.interval <= 30:
|
|
parser.error("invalid duration or interval")
|
|
path = Path(args.output)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
end = time.monotonic() + args.duration
|
|
with path.open("w", encoding="utf-8") as out:
|
|
while True:
|
|
row = snapshot(args.prometheus)
|
|
out.write(json.dumps(row, sort_keys=True) + "\n")
|
|
out.flush()
|
|
if time.monotonic() + args.interval > end:
|
|
break
|
|
time.sleep(args.interval)
|
|
print(f"saved {path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|