Files
mtusi2026lab6/student-tests/obsutil.py
T

215 lines
7.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Доступ к сохранённым наблюдениям через Grafana (единственный интерфейс к хранилищам).
Prometheus, Loki и Tempo не публикуют порты: запросы идут через datasource proxy
Grafana (http://127.0.0.1:3000/api/datasources/proxy/uid/<uid>/...) с учётной записью
admin из secrets/grafana_admin_password. Пароль не печатается.
"""
import base64
import json
import os
import time
import urllib.parse
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
GRAFANA = os.environ.get('GRAFANA_URL', 'http://127.0.0.1:3000')
def _auth():
pw = (ROOT / 'secrets' / 'grafana_admin_password').read_text().strip()
return 'Basic ' + base64.b64encode(('admin:' + pw).encode()).decode()
def grafana(path, params=None, method='GET', body=None, timeout=30):
"""Запрос к Grafana; 502/503/504 (backend перезапускается) — повтор до ~60 s."""
import urllib.error
for attempt in range(12):
try:
return _grafana(path, params, method, body, timeout)
except urllib.error.HTTPError as e:
if e.code not in (502, 503, 504) or attempt == 11:
raise
time.sleep(5)
def _grafana(path, params=None, method='GET', body=None, timeout=30):
url = GRAFANA + path
if params:
url += ('&' if '?' in url else '?') + urllib.parse.urlencode(params)
data = json.dumps(body).encode() if body is not None else None
req = urllib.request.Request(url, data=data, method=method,
headers={'Authorization': _auth(), 'Content-Type': 'application/json'})
with urllib.request.urlopen(req, timeout=timeout) as r:
raw = r.read()
return json.loads(raw) if raw else None
def wait_grafana(timeout=120):
end = time.time() + timeout
while time.time() < end:
try:
if grafana('/api/health')['database'] == 'ok':
return
except Exception:
pass
time.sleep(2)
raise RuntimeError('grafana not ready')
# ---------------- Prometheus ----------------
def prom(query, at=None):
p = {'query': query}
if at:
p['time'] = at
r = grafana('/api/datasources/proxy/uid/prometheus/api/v1/query', p)
return r['data']['result']
def prom_value(query, at=None, default=None):
res = prom(query, at)
if not res:
return default
return float(res[0]['value'][1])
def prom_range(query, start, end, step=5):
r = grafana('/api/datasources/proxy/uid/prometheus/api/v1/query_range',
{'query': query, 'start': start, 'end': end, 'step': step})
return r['data']['result']
def prom_labels():
return grafana('/api/datasources/proxy/uid/prometheus/api/v1/labels')['data']
def prom_label_values(name):
return grafana('/api/datasources/proxy/uid/prometheus/api/v1/label/%s/values' % name)['data']
def alerts():
"""Активные alerts Prometheus: {alertname: state} (pending/firing)."""
r = grafana('/api/datasources/proxy/uid/prometheus/api/v1/alerts')
return {a['labels']['alertname']: a['state'] for a in r['data']['alerts']}
# ---------------- Loki ----------------
def loki(query, start, end=None, limit=200):
"""Записи логов: [(ts_ns, line, labels)] (labels включают structured metadata)."""
end = end or time.time() + 1
r = grafana('/api/datasources/proxy/uid/loki/loki/api/v1/query_range',
{'query': query, 'start': int(start * 1e9), 'end': int(end * 1e9), 'limit': limit,
'direction': 'forward'})
out = []
for stream in r['data']['result']:
for ts, line, *rest in stream['values']:
labels = dict(stream['stream'])
if rest and isinstance(rest[0], dict):
for v in rest[0].values():
if isinstance(v, dict):
labels.update(v)
out.append((int(ts), line, labels))
out.sort(key=lambda x: x[0])
return out
# ---------------- Tempo ----------------
def tempo_search(traceql, start, end=None, limit=20):
end = end or time.time() + 1
r = grafana('/api/datasources/proxy/uid/tempo/api/search',
{'q': traceql, 'start': int(start), 'end': int(end) + 1, 'limit': limit})
return r.get('traces') or []
def _b64hex(v):
if not v:
return ''
if all(c in '0123456789abcdef' for c in v.lower()) and len(v) in (16, 32):
return v.lower()
return base64.b64decode(v).hex()
def tempo_trace(trace_id):
"""Spans трассы: [{service, name, kind, span_id, parent, start, end, ms, attrs, status}]."""
r = grafana('/api/datasources/proxy/uid/tempo/api/traces/' + trace_id)
batches = r.get('batches') or r.get('resourceSpans') or (r.get('trace') or {}).get('resourceSpans') or []
spans = []
for b in batches:
res = {a['key']: _val(a['value']) for a in b.get('resource', {}).get('attributes', [])}
for ss in b.get('scopeSpans') or b.get('instrumentationLibrarySpans') or []:
for s in ss.get('spans', []):
st, en = int(s.get('startTimeUnixNano', 0)), int(s.get('endTimeUnixNano', 0))
spans.append({
'service': res.get('service.name'), 'instance': res.get('service.instance.id'),
'name': s.get('name'), 'kind': s.get('kind', ''),
'span_id': _b64hex(s.get('spanId')), 'parent': _b64hex(s.get('parentSpanId')),
'start': st, 'end': en, 'ms': round((en - st) / 1e6, 2),
'attrs': {a['key']: _val(a['value']) for a in s.get('attributes', [])},
'status': (s.get('status') or {}).get('code', ''),
})
spans.sort(key=lambda s: s['start'])
return spans, r
def _val(v):
for k in ('stringValue', 'intValue', 'boolValue', 'doubleValue'):
if k in v:
return v[k]
return json.dumps(v)
def find_trace(traceql, start, timeout=90):
"""Ждёт появления трассы по TraceQL (ingest Collector → Tempo занимает секунды)."""
end = time.time() + timeout
while time.time() < end:
res = tempo_search(traceql, start)
if res:
return res[0]['traceID']
time.sleep(2)
return None
def wait_until(fn, timeout, interval=2):
end = time.time() + timeout
while time.time() < end:
v = fn()
if v:
return v
time.sleep(interval)
return None
# ---------------- cgroup ресурсов контейнера (с хоста) ----------------
def cgroup_dir(cid):
for p in ('/sys/fs/cgroup/system.slice/docker-%s.scope' % cid, '/sys/fs/cgroup/docker/%s' % cid):
if os.path.isdir(p):
return p
return None
def cg_read(cid, name):
d = cgroup_dir(cid)
if not d:
return None
try:
return Path(d, name).read_text()
except OSError:
return None
def cpu_usec(cid):
t = cg_read(cid, 'cpu.stat') or ''
for line in t.splitlines():
if line.startswith('usage_usec '):
return int(line.split()[1])
return None
def mem_bytes(cid, name='memory.current'):
t = cg_read(cid, name)
return int(t.strip()) if t and t.strip().isdigit() else None