51 lines
2.0 KiB
Python
51 lines
2.0 KiB
Python
#!/usr/bin/env python3
|
|
"""lab6: журнал переходов alert'ов Prometheus (inactive -> pending -> firing -> resolved).
|
|
|
|
python scripts/alerts_watch.py --duration 600 > evidence/alerts_timeline.txt
|
|
|
|
Опрашивает http://127.0.0.1:9090/api/v1/alerts и печатает строку при каждом
|
|
изменении состояния (UTC-время, имя, состояние, значение). «resolved» = alert
|
|
исчез из списка активных. Только стандартная библиотека.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from urllib.request import urlopen
|
|
|
|
|
|
def now():
|
|
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--prometheus", default="http://127.0.0.1:9090")
|
|
p.add_argument("--duration", type=float, default=600)
|
|
p.add_argument("--interval", type=float, default=5)
|
|
args = p.parse_args()
|
|
seen = {}
|
|
end = time.monotonic() + args.duration
|
|
print("%s watch start %s" % (now(), args.prometheus), flush=True)
|
|
while time.monotonic() < end:
|
|
try:
|
|
data = json.load(urlopen(args.prometheus + "/api/v1/alerts", timeout=5))["data"]["alerts"]
|
|
except Exception as exc:
|
|
print("%s prometheus unavailable: %s" % (now(), type(exc).__name__), flush=True)
|
|
time.sleep(args.interval)
|
|
continue
|
|
current = {a["labels"]["alertname"]: a for a in data}
|
|
for name, a in current.items():
|
|
if seen.get(name) != a["state"]:
|
|
print("%s %-22s %-8s value=%s active_since=%s" % (now(), name, a["state"], a.get("value"),
|
|
a.get("activeAt")), flush=True)
|
|
seen[name] = a["state"]
|
|
for name in [n for n in seen if n not in current]:
|
|
print("%s %-22s resolved" % (now(), name), flush=True)
|
|
del seen[name]
|
|
time.sleep(args.interval)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|