forked from mtusi2026/mtusi2026lab6
126 lines
3.9 KiB
Python
126 lines
3.9 KiB
Python
import json
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
from client import Failure, check
|
|
|
|
PROM = 'http://127.0.0.1:9090'
|
|
LOKI = 'http://127.0.0.1:3100'
|
|
TEMPO = 'http://127.0.0.1:3200'
|
|
|
|
|
|
def get(url, params=None, timeout=10):
|
|
if params:
|
|
url += '?' + urllib.parse.urlencode(params)
|
|
with urllib.request.urlopen(url, timeout=timeout) as response:
|
|
return json.loads(response.read())
|
|
|
|
|
|
def prom(expr):
|
|
data = get(PROM + '/api/v1/query', {'query': expr})
|
|
check(data['status'] == 'success', 'prometheus query failed: ' + expr)
|
|
return data['data']['result']
|
|
|
|
|
|
def prom_value(expr, default=None):
|
|
result = prom(expr)
|
|
if not result:
|
|
return default
|
|
value = float(result[0]['value'][1])
|
|
return default if value != value else value
|
|
|
|
|
|
def series(name):
|
|
return get(PROM + '/api/v1/series', {'match[]': name})['data']
|
|
|
|
|
|
def alerts():
|
|
return {a['labels']['alertname']: a['state'] for a in get(PROM + '/api/v1/alerts')['data']['alerts']}
|
|
|
|
|
|
def wait_alert(name, firing, timeout=120):
|
|
deadline = time.monotonic() + timeout
|
|
started = time.monotonic()
|
|
while time.monotonic() < deadline:
|
|
state = alerts().get(name)
|
|
if (state == 'firing') == firing:
|
|
return time.monotonic() - started
|
|
time.sleep(2)
|
|
raise Failure('alert %s did not become %s' % (name, 'firing' if firing else 'resolved'))
|
|
|
|
|
|
def loki(query, minutes=15, limit=200):
|
|
end = time.time()
|
|
data = get(LOKI + '/loki/api/v1/query_range', {
|
|
'query': query, 'limit': limit, 'direction': 'backward',
|
|
'start': int((end - minutes * 60) * 1e9), 'end': int((end + 60) * 1e9)})
|
|
rows = []
|
|
for stream in data['data']['result']:
|
|
for ts, line in stream['values']:
|
|
rows.append((int(ts), stream['stream'], line))
|
|
return sorted(rows, key=lambda r: r[0])
|
|
|
|
|
|
def wait_loki(query, timeout=40, **kw):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
rows = loki(query, **kw)
|
|
if rows:
|
|
return rows
|
|
time.sleep(2)
|
|
raise Failure('loki returned nothing for ' + query)
|
|
|
|
|
|
def trace(trace_id, timeout=40):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
return flatten(get(TEMPO + '/api/traces/' + trace_id))
|
|
except Exception:
|
|
time.sleep(2)
|
|
raise Failure('tempo has no trace ' + trace_id)
|
|
|
|
|
|
def attrs(items):
|
|
out = {}
|
|
for a in items:
|
|
v = a['value']
|
|
out[a['key']] = v.get('stringValue', v.get('intValue', v.get('boolValue', v.get('doubleValue'))))
|
|
return out
|
|
|
|
|
|
def flatten(data):
|
|
spans = []
|
|
for batch in data.get('batches', []):
|
|
service = attrs(batch['resource']['attributes']).get('service.name')
|
|
for scope in batch['scopeSpans']:
|
|
for s in scope['spans']:
|
|
spans.append({
|
|
'service': service, 'name': s['name'], 'kind': s['kind'], 'id': s['spanId'],
|
|
'parent': s.get('parentSpanId', ''), 'start': int(s['startTimeUnixNano']),
|
|
'duration_us': (int(s['endTimeUnixNano']) - int(s['startTimeUnixNano'])) // 1000,
|
|
'attributes': attrs(s.get('attributes', [])), 'status': s.get('status', {}).get('code', ''),
|
|
})
|
|
return sorted(spans, key=lambda s: s['start'])
|
|
|
|
|
|
def trace_search(query, limit=20, since=None):
|
|
params = {'q': query, 'limit': limit}
|
|
if since:
|
|
params.update(start=int(since), end=int(time.time()) + 60)
|
|
return get(TEMPO + '/api/search', params).get('traces', [])
|
|
|
|
|
|
def wait_trace_spans(trace_id, predicate, timeout=60):
|
|
deadline = time.monotonic() + timeout
|
|
spans = []
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
spans = trace(trace_id, 5)
|
|
except Failure:
|
|
spans = []
|
|
if spans and predicate(spans):
|
|
return spans
|
|
time.sleep(2)
|
|
raise Failure('trace %s did not reach the expected shape' % trace_id)
|