forked from mtusi2026/mtusi2026lab6
224 lines
9.2 KiB
Python
224 lines
9.2 KiB
Python
"""Запросы к хранилищам телеметрии для тестов lab6 (только стандартная библиотека).
|
|
|
|
Адреса — порты, опубликованные compose.yaml только на 127.0.0.1:
|
|
PROMETHEUS_URL (http://127.0.0.1:9090), LOKI_URL (http://127.0.0.1:3100), TEMPO_URL (http://127.0.0.1:3200).
|
|
"""
|
|
import base64
|
|
import binascii
|
|
import json
|
|
import os
|
|
import re
|
|
import secrets
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
PROMETHEUS_URL = os.environ.get("PROMETHEUS_URL", "http://127.0.0.1:9090").rstrip("/")
|
|
LOKI_URL = os.environ.get("LOKI_URL", "http://127.0.0.1:3100").rstrip("/")
|
|
TEMPO_URL = os.environ.get("TEMPO_URL", "http://127.0.0.1:3200").rstrip("/")
|
|
|
|
|
|
def get_json(url, timeout=10):
|
|
request = urllib.request.Request(url, headers={"Accept": "application/json"})
|
|
with urllib.request.urlopen(request, timeout=timeout) as reply:
|
|
return json.loads(reply.read().decode("utf-8"))
|
|
|
|
|
|
# ---------- Prometheus ----------
|
|
|
|
def prom_query(expr, at=None):
|
|
"""Мгновенный запрос. Возвращает список (labels, float). Ошибка запроса -> исключение."""
|
|
params = {"query": expr}
|
|
if at is not None:
|
|
params["time"] = str(at)
|
|
data = get_json(f"{PROMETHEUS_URL}/api/v1/query?" + urllib.parse.urlencode(params))
|
|
if data.get("status") != "success":
|
|
raise RuntimeError(f"PromQL error: {data}")
|
|
result = data["data"]["result"]
|
|
if data["data"]["resultType"] == "scalar":
|
|
return [({}, float(result[1]))]
|
|
return [(item["metric"], float(item["value"][1])) for item in result]
|
|
|
|
|
|
def prom_value(expr, default=None):
|
|
rows = prom_query(expr)
|
|
return rows[0][1] if rows else default
|
|
|
|
|
|
def prom_series(match):
|
|
data = get_json(f"{PROMETHEUS_URL}/api/v1/series?" + urllib.parse.urlencode({"match[]": match}))
|
|
return data["data"]
|
|
|
|
|
|
def alerts():
|
|
"""Активные алерты: {имя: состояние (pending/firing)}."""
|
|
data = get_json(f"{PROMETHEUS_URL}/api/v1/alerts")
|
|
out = {}
|
|
for alert in data["data"]["alerts"]:
|
|
name = alert["labels"]["alertname"]
|
|
if out.get(name) != "firing":
|
|
out[name] = alert["state"]
|
|
return out
|
|
|
|
|
|
def wait_alert(name, state, seconds, poll=5, log=print):
|
|
"""Ждёт, пока алерт name перейдёт в state ('firing' или 'resolved' = отсутствует). Возвращает секунды."""
|
|
started = time.monotonic()
|
|
while True:
|
|
current = alerts().get(name, "inactive")
|
|
done = current == "firing" if state == "firing" else current == "inactive"
|
|
if done:
|
|
return round(time.monotonic() - started, 1)
|
|
if time.monotonic() - started > seconds:
|
|
raise AssertionError(f"alert {name}: still {current} after {seconds} s (want {state})")
|
|
log(f" ... {name}: {current} ({time.monotonic() - started:.0f} s)")
|
|
time.sleep(poll)
|
|
|
|
|
|
def wait_metric(expr, predicate, seconds, poll=5):
|
|
"""Ждёт, пока значение выражения удовлетворит predicate. Возвращает значение."""
|
|
deadline = time.monotonic() + seconds
|
|
while True:
|
|
value = prom_value(expr)
|
|
if value is not None and predicate(value):
|
|
return value
|
|
if time.monotonic() > deadline:
|
|
raise AssertionError(f"{expr}: value {value} after {seconds} s")
|
|
time.sleep(poll)
|
|
|
|
|
|
# ---------- Loki ----------
|
|
|
|
def loki_query(logql, minutes=30, limit=200):
|
|
"""Записи за последние minutes минут: список (labels+structured metadata, строка)."""
|
|
now = time.time_ns()
|
|
params = {"query": logql, "start": str(now - minutes * 60 * 10**9), "end": str(now), "limit": str(limit),
|
|
"direction": "backward"}
|
|
data = get_json(f"{LOKI_URL}/loki/api/v1/query_range?" + urllib.parse.urlencode(params))
|
|
if data.get("status") != "success":
|
|
raise RuntimeError(f"LogQL error: {data}")
|
|
out = []
|
|
for stream in data["data"]["result"]:
|
|
for value in stream["values"]:
|
|
labels = dict(stream["stream"])
|
|
if len(value) > 2 and isinstance(value[2], dict):
|
|
for group in value[2].values():
|
|
if isinstance(group, dict):
|
|
labels.update(group)
|
|
out.append((labels, value[1]))
|
|
return out
|
|
|
|
|
|
def wait_logs(logql, minimum=1, seconds=60):
|
|
deadline = time.monotonic() + seconds
|
|
while True:
|
|
rows = loki_query(logql)
|
|
if len(rows) >= minimum:
|
|
return rows
|
|
if time.monotonic() > deadline:
|
|
raise AssertionError(f"Loki: {len(rows)} records for {logql} after {seconds} s")
|
|
time.sleep(2)
|
|
|
|
|
|
# ---------- Tempo ----------
|
|
|
|
def new_traceparent():
|
|
"""Свой trace_id для контролируемого запроса: по нему трасса находится без поиска."""
|
|
trace_id = secrets.token_hex(16)
|
|
return trace_id, f"00-{trace_id}-{secrets.token_hex(8)}-01"
|
|
|
|
|
|
def tempo_trace(trace_id):
|
|
"""Спаны трассы: список dict(service, name, kind, span_id, parent_id, duration_ms, status, attrs, links)."""
|
|
try:
|
|
data = get_json(f"{TEMPO_URL}/api/traces/{trace_id}")
|
|
except urllib.error.HTTPError as error:
|
|
if error.code == 404:
|
|
return []
|
|
raise
|
|
spans = []
|
|
for batch in data.get("batches", data.get("resourceSpans", [])):
|
|
resource = {a["key"]: _value(a["value"]) for a in batch["resource"].get("attributes", [])}
|
|
for scope in batch.get("scopeSpans", batch.get("instrumentationLibrarySpans", [])):
|
|
for span in scope.get("spans", []):
|
|
spans.append({
|
|
"service": resource.get("service.name"),
|
|
"instance": resource.get("service.instance.id"),
|
|
"name": span["name"],
|
|
"kind": span.get("kind", "").replace("SPAN_KIND_", ""),
|
|
"span_id": _hex_id(span["spanId"], 16),
|
|
"parent_id": _hex_id(span.get("parentSpanId", ""), 16),
|
|
"start": int(span["startTimeUnixNano"]),
|
|
"duration_ms": (int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"])) / 1e6,
|
|
"status": span.get("status", {}).get("code", "").replace("STATUS_CODE_", ""),
|
|
"attrs": {a["key"]: _value(a["value"]) for a in span.get("attributes", [])},
|
|
"links": [{"trace_id": _hex_id(link.get("traceId", ""), 32),
|
|
"span_id": _hex_id(link.get("spanId", ""), 16)} for link in span.get("links", [])],
|
|
})
|
|
return spans
|
|
|
|
|
|
def _hex_id(value, size):
|
|
"""ID спана/трассы из JSON Tempo: hex или base64 (OTLP JSON кодирует bytes в base64) -> hex."""
|
|
if not value:
|
|
return ""
|
|
if re.fullmatch(r"[0-9a-fA-F]+", value) and len(value) in (size, size // 2 * 2):
|
|
return value.lower().rjust(size, "0")
|
|
try:
|
|
return base64.b64decode(value).hex().rjust(size, "0")
|
|
except (binascii.Error, ValueError):
|
|
return value
|
|
|
|
|
|
def _value(value):
|
|
for key in ("stringValue", "intValue", "boolValue", "doubleValue"):
|
|
if key in value:
|
|
return int(value[key]) if key == "intValue" else value[key]
|
|
return json.dumps(value)
|
|
|
|
|
|
def wait_trace(trace_id, predicate=lambda spans: bool(spans), seconds=60):
|
|
deadline = time.monotonic() + seconds
|
|
while True:
|
|
spans = tempo_trace(trace_id)
|
|
if predicate(spans):
|
|
return spans
|
|
if time.monotonic() > deadline:
|
|
raise AssertionError(f"Tempo: trace {trace_id} incomplete after {seconds} s ({len(spans)} spans)")
|
|
time.sleep(2)
|
|
|
|
|
|
def tempo_search(traceql, minutes=30, limit=20):
|
|
now = int(time.time())
|
|
params = {"q": traceql, "start": str(now - minutes * 60), "end": str(now), "limit": str(limit)}
|
|
return get_json(f"{TEMPO_URL}/api/search?" + urllib.parse.urlencode(params)).get("traces", [])
|
|
|
|
|
|
def span_tree(spans):
|
|
"""Текстовое дерево трассы для отчёта (без атрибутов с данными пользователя)."""
|
|
by_id = {s["span_id"]: s for s in spans}
|
|
children = {}
|
|
for span in spans:
|
|
parent = span["parent_id"] if span["parent_id"] in by_id else ""
|
|
children.setdefault(parent, []).append(span)
|
|
lines = []
|
|
|
|
def walk(span, depth):
|
|
extra = []
|
|
for key in ("messenger.job.attempt", "messenger.job.outcome", "http.response.status_code", "error.type",
|
|
"aws.s3.key"):
|
|
if key in span["attrs"]:
|
|
extra.append(f"{key}={span['attrs'][key]}")
|
|
status = " ERROR" if span["status"] == "ERROR" else ""
|
|
links = f" links={len(span['links'])}" if span["links"] else ""
|
|
short = (span["service"] or "?").replace("messenger-", "")
|
|
lines.append(f"{' ' * depth}- [{short}] {span['name']} ({span['kind']}, {span['duration_ms']:.1f} ms)"
|
|
f"{status}{links} {' '.join(extra)}".rstrip())
|
|
for child in sorted(children.get(span["span_id"], []), key=lambda s: s["start"]):
|
|
walk(child, depth + 1)
|
|
|
|
for root in sorted(children.get("", []), key=lambda s: s["start"]):
|
|
walk(root, 0)
|
|
return "\n".join(lines)
|