Files
mtusi2026lab6-fork/tests/obs.py
T

126 lines
3.9 KiB
Python

import json
import time
import urllib.parse
import urllib.request
from client import Failure, check
PROM = 'http://127.0.0.1:9090'
LOKI = 'http://127.0.0.1:3100'
TEMPO = 'http://127.0.0.1:3200'
def get(url, params=None, timeout=10):
if params:
url += '?' + urllib.parse.urlencode(params)
with urllib.request.urlopen(url, timeout=timeout) as response:
return json.loads(response.read())
def prom(expr):
data = get(PROM + '/api/v1/query', {'query': expr})
check(data['status'] == 'success', 'prometheus query failed: ' + expr)
return data['data']['result']
def prom_value(expr, default=None):
result = prom(expr)
if not result:
return default
value = float(result[0]['value'][1])
return default if value != value else value
def series(name):
return get(PROM + '/api/v1/series', {'match[]': name})['data']
def alerts():
return {a['labels']['alertname']: a['state'] for a in get(PROM + '/api/v1/alerts')['data']['alerts']}
def wait_alert(name, firing, timeout=120):
deadline = time.monotonic() + timeout
started = time.monotonic()
while time.monotonic() < deadline:
state = alerts().get(name)
if (state == 'firing') == firing:
return time.monotonic() - started
time.sleep(2)
raise Failure('alert %s did not become %s' % (name, 'firing' if firing else 'resolved'))
def loki(query, minutes=15, limit=200):
end = time.time()
data = get(LOKI + '/loki/api/v1/query_range', {
'query': query, 'limit': limit, 'direction': 'backward',
'start': int((end - minutes * 60) * 1e9), 'end': int((end + 60) * 1e9)})
rows = []
for stream in data['data']['result']:
for ts, line in stream['values']:
rows.append((int(ts), stream['stream'], line))
return sorted(rows, key=lambda r: r[0])
def wait_loki(query, timeout=40, **kw):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
rows = loki(query, **kw)
if rows:
return rows
time.sleep(2)
raise Failure('loki returned nothing for ' + query)
def trace(trace_id, timeout=40):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
return flatten(get(TEMPO + '/api/traces/' + trace_id))
except Exception:
time.sleep(2)
raise Failure('tempo has no trace ' + trace_id)
def attrs(items):
out = {}
for a in items:
v = a['value']
out[a['key']] = v.get('stringValue', v.get('intValue', v.get('boolValue', v.get('doubleValue'))))
return out
def flatten(data):
spans = []
for batch in data.get('batches', []):
service = attrs(batch['resource']['attributes']).get('service.name')
for scope in batch['scopeSpans']:
for s in scope['spans']:
spans.append({
'service': service, 'name': s['name'], 'kind': s['kind'], 'id': s['spanId'],
'parent': s.get('parentSpanId', ''), 'start': int(s['startTimeUnixNano']),
'duration_us': (int(s['endTimeUnixNano']) - int(s['startTimeUnixNano'])) // 1000,
'attributes': attrs(s.get('attributes', [])), 'status': s.get('status', {}).get('code', ''),
})
return sorted(spans, key=lambda s: s['start'])
def trace_search(query, limit=20, since=None):
params = {'q': query, 'limit': limit}
if since:
params.update(start=int(since), end=int(time.time()) + 60)
return get(TEMPO + '/api/search', params).get('traces', [])
def wait_trace_spans(trace_id, predicate, timeout=60):
deadline = time.monotonic() + timeout
spans = []
while time.monotonic() < deadline:
try:
spans = trace(trace_id, 5)
except Failure:
spans = []
if spans and predicate(spans):
return spans
time.sleep(2)
raise Failure('trace %s did not reach the expected shape' % trace_id)