feat: Add live dashboard and freeze SLO alerts (U3/U4 setup)
This commit is contained in:
@@ -30,7 +30,7 @@ def main() -> None:
|
||||
write("postgres_password", password)
|
||||
write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger")
|
||||
if not (ROOT / "minio_root_user").exists():
|
||||
write("minio_root_user", "lab5admin")
|
||||
write("minio_root_user", "lab6admin")
|
||||
write("minio_root_password", secrets.token_urlsafe(32))
|
||||
write("s3_access_key_id", secrets.token_hex(10))
|
||||
write("s3_secret_access_key", secrets.token_urlsafe(32))
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Capture local Prometheus alert states and SLI values without credentials."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import urlopen
|
||||
|
||||
|
||||
QUERIES = {
|
||||
"http_5xx_ratio_30s": 'sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])),0.001)',
|
||||
"queue_oldest_seconds": 'max(messenger_queue_oldest_age_seconds{queue_name="attachments.process"})',
|
||||
"ready_slow_ratio_30s": '(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])) - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[30s]))) / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])),0.001)',
|
||||
"collector_failed_spans": 'sum(otelcol_exporter_send_failed_spans_total)',
|
||||
}
|
||||
|
||||
|
||||
def fetch(base, path, params=None):
|
||||
url = base + path + ("?" + urlencode(params) if params else "")
|
||||
with urlopen(url, timeout=4) as response:
|
||||
result = json.load(response)
|
||||
if result["status"] != "success":
|
||||
raise RuntimeError(result)
|
||||
return result["data"]
|
||||
|
||||
|
||||
def snapshot(base):
|
||||
groups = fetch(base, "/api/v1/rules", {"type": "alert"})["groups"]
|
||||
alerts = [rule for group in groups for rule in group["rules"]]
|
||||
row = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||
"alerts": {item["name"]: item["state"]
|
||||
for item in alerts if item["name"].startswith("Lab6")},
|
||||
"values": {},
|
||||
}
|
||||
for name, query in QUERIES.items():
|
||||
values = fetch(base, "/api/v1/query", {"query": query})["result"]
|
||||
row["values"][name] = float(values[0]["value"][1]) if values else None
|
||||
return row
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--prometheus", default="http://127.0.0.1:9090")
|
||||
parser.add_argument("--duration", type=int, default=60)
|
||||
parser.add_argument("--interval", type=int, default=5)
|
||||
parser.add_argument("--output", required=True)
|
||||
args = parser.parse_args()
|
||||
if not 5 <= args.duration <= 900 or not 1 <= args.interval <= 30:
|
||||
parser.error("invalid duration or interval")
|
||||
path = Path(args.output)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
end = time.monotonic() + args.duration
|
||||
with path.open("w", encoding="utf-8") as out:
|
||||
while True:
|
||||
row = snapshot(args.prometheus)
|
||||
out.write(json.dumps(row, sort_keys=True) + "\n")
|
||||
out.flush()
|
||||
if time.monotonic() + args.interval > end:
|
||||
break
|
||||
time.sleep(args.interval)
|
||||
print(f"saved {path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user