feat: Add live dashboard and freeze SLO alerts (U3/U4 setup)

This commit is contained in:
Самохвалов Андрей Ремович
2026-10-02 15:32:51 +03:00
parent 9fac497d1a
commit 5deecdf545
15 changed files with 976 additions and 22 deletions
+1 -1
View File
@@ -30,7 +30,7 @@ def main() -> None:
write("postgres_password", password)
write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger")
if not (ROOT / "minio_root_user").exists():
write("minio_root_user", "lab5admin")
write("minio_root_user", "lab6admin")
write("minio_root_password", secrets.token_urlsafe(32))
write("s3_access_key_id", secrets.token_hex(10))
write("s3_secret_access_key", secrets.token_urlsafe(32))
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env python3
"""Capture local Prometheus alert states and SLI values without credentials."""
import argparse
import json
import time
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urlencode
from urllib.request import urlopen
QUERIES = {
"http_5xx_ratio_30s": 'sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])),0.001)',
"queue_oldest_seconds": 'max(messenger_queue_oldest_age_seconds{queue_name="attachments.process"})',
"ready_slow_ratio_30s": '(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])) - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[30s]))) / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])),0.001)',
"collector_failed_spans": 'sum(otelcol_exporter_send_failed_spans_total)',
}
def fetch(base, path, params=None):
url = base + path + ("?" + urlencode(params) if params else "")
with urlopen(url, timeout=4) as response:
result = json.load(response)
if result["status"] != "success":
raise RuntimeError(result)
return result["data"]
def snapshot(base):
groups = fetch(base, "/api/v1/rules", {"type": "alert"})["groups"]
alerts = [rule for group in groups for rule in group["rules"]]
row = {
"timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"alerts": {item["name"]: item["state"]
for item in alerts if item["name"].startswith("Lab6")},
"values": {},
}
for name, query in QUERIES.items():
values = fetch(base, "/api/v1/query", {"query": query})["result"]
row["values"][name] = float(values[0]["value"][1]) if values else None
return row
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--prometheus", default="http://127.0.0.1:9090")
parser.add_argument("--duration", type=int, default=60)
parser.add_argument("--interval", type=int, default=5)
parser.add_argument("--output", required=True)
args = parser.parse_args()
if not 5 <= args.duration <= 900 or not 1 <= args.interval <= 30:
parser.error("invalid duration or interval")
path = Path(args.output)
path.parent.mkdir(parents=True, exist_ok=True)
end = time.monotonic() + args.duration
with path.open("w", encoding="utf-8") as out:
while True:
row = snapshot(args.prometheus)
out.write(json.dumps(row, sort_keys=True) + "\n")
out.flush()
if time.monotonic() + args.interval > end:
break
time.sleep(args.interval)
print(f"saved {path}")
if __name__ == "__main__":
main()