Files

74 lines
2.8 KiB
Python
Executable File

#!/usr/bin/env python3
"""Poll Prometheus alert state and write a firing/resolved timeline."""
import argparse
import os
import csv
import json
import time
import urllib.parse
import urllib.request
from pathlib import Path
from lab7_http import open_url
# The lab tools always target loopback; ignore ambient corporate proxies.
for _proxy_key in ('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'http_proxy', 'https_proxy', 'all_proxy'):
os.environ.pop(_proxy_key, None)
os.environ['NO_PROXY'] = 'localhost,127.0.0.1,::1'
os.environ['no_proxy'] = 'localhost,127.0.0.1,::1'
def get(url):
with open_url(url, timeout=5, operator=True) as response:
return json.loads(response.read())
def main():
parser = argparse.ArgumentParser(description='Poll Prometheus alerts')
parser.add_argument('--prometheus', default='https://localhost:9090')
parser.add_argument('--duration', type=float, default=600)
parser.add_argument('--interval', type=float, default=5)
parser.add_argument('--output', default='evidence/lab6/alerts.csv')
parser.add_argument('--injection-utc', default='')
args = parser.parse_args()
path = Path(args.output)
path.parent.mkdir(parents=True, exist_ok=True)
started = time.time()
active = set()
had_error = False
with path.open('w', newline='') as handle:
writer = csv.writer(handle)
writer.writerow(['observed_utc', 'alert', 'state', 'active_at', 'value', 'injection_utc'])
while time.time() - started < args.duration:
try:
payload = get(args.prometheus.rstrip('/') + '/api/v1/alerts')
current = set()
for alert in payload.get('data', {}).get('alerts', []):
name = alert.get('labels', {}).get('alertname', '')
state = alert.get('state', '')
current.add((name, state))
if (name, state) not in active:
writer.writerow([
time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
name, state, alert.get('activeAt', ''),
json.dumps(alert.get('value', ''), separators=(',', ':')), args.injection_utc,
])
for name, state in sorted(active - current):
writer.writerow([
time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
name, 'resolved', '', '', args.injection_utc,
])
active = current
handle.flush()
except Exception as exc:
had_error = True
print('WARN: alert poll failed:', exc)
time.sleep(args.interval)
print('alert timeline:', path)
return 1 if had_error else 0
if __name__ == '__main__':
raise SystemExit(main())