"""Доступ к сохранённым наблюдениям через Grafana (единственный интерфейс к хранилищам). Prometheus, Loki и Tempo не публикуют порты: запросы идут через datasource proxy Grafana (http://127.0.0.1:3000/api/datasources/proxy/uid//...) с учётной записью admin из secrets/grafana_admin_password. Пароль не печатается. """ import base64 import json import os import time import urllib.parse import urllib.request from pathlib import Path ROOT = Path(__file__).resolve().parents[1] GRAFANA = os.environ.get('GRAFANA_URL', 'http://127.0.0.1:3000') def _auth(): pw = (ROOT / 'secrets' / 'grafana_admin_password').read_text().strip() return 'Basic ' + base64.b64encode(('admin:' + pw).encode()).decode() def grafana(path, params=None, method='GET', body=None, timeout=30): """Запрос к Grafana; 502/503/504 (backend перезапускается) — повтор до ~60 s.""" import urllib.error for attempt in range(12): try: return _grafana(path, params, method, body, timeout) except urllib.error.HTTPError as e: if e.code not in (502, 503, 504) or attempt == 11: raise time.sleep(5) def _grafana(path, params=None, method='GET', body=None, timeout=30): url = GRAFANA + path if params: url += ('&' if '?' in url else '?') + urllib.parse.urlencode(params) data = json.dumps(body).encode() if body is not None else None req = urllib.request.Request(url, data=data, method=method, headers={'Authorization': _auth(), 'Content-Type': 'application/json'}) with urllib.request.urlopen(req, timeout=timeout) as r: raw = r.read() return json.loads(raw) if raw else None def wait_grafana(timeout=120): end = time.time() + timeout while time.time() < end: try: if grafana('/api/health')['database'] == 'ok': return except Exception: pass time.sleep(2) raise RuntimeError('grafana not ready') # ---------------- Prometheus ---------------- def prom(query, at=None): p = {'query': query} if at: p['time'] = at r = grafana('/api/datasources/proxy/uid/prometheus/api/v1/query', p) return r['data']['result'] def prom_value(query, at=None, default=None): res = prom(query, at) if not res: return default return float(res[0]['value'][1]) def prom_range(query, start, end, step=5): r = grafana('/api/datasources/proxy/uid/prometheus/api/v1/query_range', {'query': query, 'start': start, 'end': end, 'step': step}) return r['data']['result'] def prom_labels(): return grafana('/api/datasources/proxy/uid/prometheus/api/v1/labels')['data'] def prom_label_values(name): return grafana('/api/datasources/proxy/uid/prometheus/api/v1/label/%s/values' % name)['data'] def alerts(): """Активные alerts Prometheus: {alertname: state} (pending/firing).""" r = grafana('/api/datasources/proxy/uid/prometheus/api/v1/alerts') return {a['labels']['alertname']: a['state'] for a in r['data']['alerts']} # ---------------- Loki ---------------- def loki(query, start, end=None, limit=200): """Записи логов: [(ts_ns, line, labels)] (labels включают structured metadata).""" end = end or time.time() + 1 r = grafana('/api/datasources/proxy/uid/loki/loki/api/v1/query_range', {'query': query, 'start': int(start * 1e9), 'end': int(end * 1e9), 'limit': limit, 'direction': 'forward'}) out = [] for stream in r['data']['result']: for ts, line, *rest in stream['values']: labels = dict(stream['stream']) if rest and isinstance(rest[0], dict): for v in rest[0].values(): if isinstance(v, dict): labels.update(v) out.append((int(ts), line, labels)) out.sort(key=lambda x: x[0]) return out # ---------------- Tempo ---------------- def tempo_search(traceql, start, end=None, limit=20): end = end or time.time() + 1 r = grafana('/api/datasources/proxy/uid/tempo/api/search', {'q': traceql, 'start': int(start), 'end': int(end) + 1, 'limit': limit}) return r.get('traces') or [] def _b64hex(v): if not v: return '' if all(c in '0123456789abcdef' for c in v.lower()) and len(v) in (16, 32): return v.lower() return base64.b64decode(v).hex() def tempo_trace(trace_id): """Spans трассы: [{service, name, kind, span_id, parent, start, end, ms, attrs, status}].""" r = grafana('/api/datasources/proxy/uid/tempo/api/traces/' + trace_id) batches = r.get('batches') or r.get('resourceSpans') or (r.get('trace') or {}).get('resourceSpans') or [] spans = [] for b in batches: res = {a['key']: _val(a['value']) for a in b.get('resource', {}).get('attributes', [])} for ss in b.get('scopeSpans') or b.get('instrumentationLibrarySpans') or []: for s in ss.get('spans', []): st, en = int(s.get('startTimeUnixNano', 0)), int(s.get('endTimeUnixNano', 0)) spans.append({ 'service': res.get('service.name'), 'instance': res.get('service.instance.id'), 'name': s.get('name'), 'kind': s.get('kind', ''), 'span_id': _b64hex(s.get('spanId')), 'parent': _b64hex(s.get('parentSpanId')), 'start': st, 'end': en, 'ms': round((en - st) / 1e6, 2), 'attrs': {a['key']: _val(a['value']) for a in s.get('attributes', [])}, 'status': (s.get('status') or {}).get('code', ''), }) spans.sort(key=lambda s: s['start']) return spans, r def _val(v): for k in ('stringValue', 'intValue', 'boolValue', 'doubleValue'): if k in v: return v[k] return json.dumps(v) def find_trace(traceql, start, timeout=90): """Ждёт появления трассы по TraceQL (ingest Collector → Tempo занимает секунды).""" end = time.time() + timeout while time.time() < end: res = tempo_search(traceql, start) if res: return res[0]['traceID'] time.sleep(2) return None def wait_until(fn, timeout, interval=2): end = time.time() + timeout while time.time() < end: v = fn() if v: return v time.sleep(interval) return None # ---------------- cgroup ресурсов контейнера (с хоста) ---------------- def cgroup_dir(cid): for p in ('/sys/fs/cgroup/system.slice/docker-%s.scope' % cid, '/sys/fs/cgroup/docker/%s' % cid): if os.path.isdir(p): return p return None def cg_read(cid, name): d = cgroup_dir(cid) if not d: return None try: return Path(d, name).read_text() except OSError: return None def cpu_usec(cid): t = cg_read(cid, 'cpu.stat') or '' for line in t.splitlines(): if line.startswith('usage_usec '): return int(line.split()[1]) return None def mem_bytes(cid, name='memory.current'): t = cg_read(cid, name) return int(t.strip()) if t and t.strip().isdigit() else None