215 lines
7.3 KiB
Python
215 lines
7.3 KiB
Python
"""Доступ к сохранённым наблюдениям через Grafana (единственный интерфейс к хранилищам).
|
||
|
||
Prometheus, Loki и Tempo не публикуют порты: запросы идут через datasource proxy
|
||
Grafana (http://127.0.0.1:3000/api/datasources/proxy/uid/<uid>/...) с учётной записью
|
||
admin из secrets/grafana_admin_password. Пароль не печатается.
|
||
"""
|
||
import base64
|
||
import json
|
||
import os
|
||
import time
|
||
import urllib.parse
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
GRAFANA = os.environ.get('GRAFANA_URL', 'http://127.0.0.1:3000')
|
||
|
||
|
||
def _auth():
|
||
pw = (ROOT / 'secrets' / 'grafana_admin_password').read_text().strip()
|
||
return 'Basic ' + base64.b64encode(('admin:' + pw).encode()).decode()
|
||
|
||
|
||
def grafana(path, params=None, method='GET', body=None, timeout=30):
|
||
"""Запрос к Grafana; 502/503/504 (backend перезапускается) — повтор до ~60 s."""
|
||
import urllib.error
|
||
for attempt in range(12):
|
||
try:
|
||
return _grafana(path, params, method, body, timeout)
|
||
except urllib.error.HTTPError as e:
|
||
if e.code not in (502, 503, 504) or attempt == 11:
|
||
raise
|
||
time.sleep(5)
|
||
|
||
|
||
def _grafana(path, params=None, method='GET', body=None, timeout=30):
|
||
url = GRAFANA + path
|
||
if params:
|
||
url += ('&' if '?' in url else '?') + urllib.parse.urlencode(params)
|
||
data = json.dumps(body).encode() if body is not None else None
|
||
req = urllib.request.Request(url, data=data, method=method,
|
||
headers={'Authorization': _auth(), 'Content-Type': 'application/json'})
|
||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||
raw = r.read()
|
||
return json.loads(raw) if raw else None
|
||
|
||
|
||
def wait_grafana(timeout=120):
|
||
end = time.time() + timeout
|
||
while time.time() < end:
|
||
try:
|
||
if grafana('/api/health')['database'] == 'ok':
|
||
return
|
||
except Exception:
|
||
pass
|
||
time.sleep(2)
|
||
raise RuntimeError('grafana not ready')
|
||
|
||
|
||
# ---------------- Prometheus ----------------
|
||
|
||
def prom(query, at=None):
|
||
p = {'query': query}
|
||
if at:
|
||
p['time'] = at
|
||
r = grafana('/api/datasources/proxy/uid/prometheus/api/v1/query', p)
|
||
return r['data']['result']
|
||
|
||
|
||
def prom_value(query, at=None, default=None):
|
||
res = prom(query, at)
|
||
if not res:
|
||
return default
|
||
return float(res[0]['value'][1])
|
||
|
||
|
||
def prom_range(query, start, end, step=5):
|
||
r = grafana('/api/datasources/proxy/uid/prometheus/api/v1/query_range',
|
||
{'query': query, 'start': start, 'end': end, 'step': step})
|
||
return r['data']['result']
|
||
|
||
|
||
def prom_labels():
|
||
return grafana('/api/datasources/proxy/uid/prometheus/api/v1/labels')['data']
|
||
|
||
|
||
def prom_label_values(name):
|
||
return grafana('/api/datasources/proxy/uid/prometheus/api/v1/label/%s/values' % name)['data']
|
||
|
||
|
||
def alerts():
|
||
"""Активные alerts Prometheus: {alertname: state} (pending/firing)."""
|
||
r = grafana('/api/datasources/proxy/uid/prometheus/api/v1/alerts')
|
||
return {a['labels']['alertname']: a['state'] for a in r['data']['alerts']}
|
||
|
||
|
||
# ---------------- Loki ----------------
|
||
|
||
def loki(query, start, end=None, limit=200):
|
||
"""Записи логов: [(ts_ns, line, labels)] (labels включают structured metadata)."""
|
||
end = end or time.time() + 1
|
||
r = grafana('/api/datasources/proxy/uid/loki/loki/api/v1/query_range',
|
||
{'query': query, 'start': int(start * 1e9), 'end': int(end * 1e9), 'limit': limit,
|
||
'direction': 'forward'})
|
||
out = []
|
||
for stream in r['data']['result']:
|
||
for ts, line, *rest in stream['values']:
|
||
labels = dict(stream['stream'])
|
||
if rest and isinstance(rest[0], dict):
|
||
for v in rest[0].values():
|
||
if isinstance(v, dict):
|
||
labels.update(v)
|
||
out.append((int(ts), line, labels))
|
||
out.sort(key=lambda x: x[0])
|
||
return out
|
||
|
||
|
||
# ---------------- Tempo ----------------
|
||
|
||
def tempo_search(traceql, start, end=None, limit=20):
|
||
end = end or time.time() + 1
|
||
r = grafana('/api/datasources/proxy/uid/tempo/api/search',
|
||
{'q': traceql, 'start': int(start), 'end': int(end) + 1, 'limit': limit})
|
||
return r.get('traces') or []
|
||
|
||
|
||
def _b64hex(v):
|
||
if not v:
|
||
return ''
|
||
if all(c in '0123456789abcdef' for c in v.lower()) and len(v) in (16, 32):
|
||
return v.lower()
|
||
return base64.b64decode(v).hex()
|
||
|
||
|
||
def tempo_trace(trace_id):
|
||
"""Spans трассы: [{service, name, kind, span_id, parent, start, end, ms, attrs, status}]."""
|
||
r = grafana('/api/datasources/proxy/uid/tempo/api/traces/' + trace_id)
|
||
batches = r.get('batches') or r.get('resourceSpans') or (r.get('trace') or {}).get('resourceSpans') or []
|
||
spans = []
|
||
for b in batches:
|
||
res = {a['key']: _val(a['value']) for a in b.get('resource', {}).get('attributes', [])}
|
||
for ss in b.get('scopeSpans') or b.get('instrumentationLibrarySpans') or []:
|
||
for s in ss.get('spans', []):
|
||
st, en = int(s.get('startTimeUnixNano', 0)), int(s.get('endTimeUnixNano', 0))
|
||
spans.append({
|
||
'service': res.get('service.name'), 'instance': res.get('service.instance.id'),
|
||
'name': s.get('name'), 'kind': s.get('kind', ''),
|
||
'span_id': _b64hex(s.get('spanId')), 'parent': _b64hex(s.get('parentSpanId')),
|
||
'start': st, 'end': en, 'ms': round((en - st) / 1e6, 2),
|
||
'attrs': {a['key']: _val(a['value']) for a in s.get('attributes', [])},
|
||
'status': (s.get('status') or {}).get('code', ''),
|
||
})
|
||
spans.sort(key=lambda s: s['start'])
|
||
return spans, r
|
||
|
||
|
||
def _val(v):
|
||
for k in ('stringValue', 'intValue', 'boolValue', 'doubleValue'):
|
||
if k in v:
|
||
return v[k]
|
||
return json.dumps(v)
|
||
|
||
|
||
def find_trace(traceql, start, timeout=90):
|
||
"""Ждёт появления трассы по TraceQL (ingest Collector → Tempo занимает секунды)."""
|
||
end = time.time() + timeout
|
||
while time.time() < end:
|
||
res = tempo_search(traceql, start)
|
||
if res:
|
||
return res[0]['traceID']
|
||
time.sleep(2)
|
||
return None
|
||
|
||
|
||
def wait_until(fn, timeout, interval=2):
|
||
end = time.time() + timeout
|
||
while time.time() < end:
|
||
v = fn()
|
||
if v:
|
||
return v
|
||
time.sleep(interval)
|
||
return None
|
||
|
||
|
||
# ---------------- cgroup ресурсов контейнера (с хоста) ----------------
|
||
|
||
def cgroup_dir(cid):
|
||
for p in ('/sys/fs/cgroup/system.slice/docker-%s.scope' % cid, '/sys/fs/cgroup/docker/%s' % cid):
|
||
if os.path.isdir(p):
|
||
return p
|
||
return None
|
||
|
||
|
||
def cg_read(cid, name):
|
||
d = cgroup_dir(cid)
|
||
if not d:
|
||
return None
|
||
try:
|
||
return Path(d, name).read_text()
|
||
except OSError:
|
||
return None
|
||
|
||
|
||
def cpu_usec(cid):
|
||
t = cg_read(cid, 'cpu.stat') or ''
|
||
for line in t.splitlines():
|
||
if line.startswith('usage_usec '):
|
||
return int(line.split()[1])
|
||
return None
|
||
|
||
|
||
def mem_bytes(cid, name='memory.current'):
|
||
t = cg_read(cid, name)
|
||
return int(t.strip()) if t and t.strip().isdigit() else None
|