Files
mtusi2026lab6/student-tests/obs.py

224 lines
9.2 KiB
Python

"""Запросы к хранилищам телеметрии для тестов lab6 (только стандартная библиотека).
Адреса — порты, опубликованные compose.yaml только на 127.0.0.1:
PROMETHEUS_URL (http://127.0.0.1:9090), LOKI_URL (http://127.0.0.1:3100), TEMPO_URL (http://127.0.0.1:3200).
"""
import base64
import binascii
import json
import os
import re
import secrets
import time
import urllib.error
import urllib.parse
import urllib.request
PROMETHEUS_URL = os.environ.get("PROMETHEUS_URL", "http://127.0.0.1:9090").rstrip("/")
LOKI_URL = os.environ.get("LOKI_URL", "http://127.0.0.1:3100").rstrip("/")
TEMPO_URL = os.environ.get("TEMPO_URL", "http://127.0.0.1:3200").rstrip("/")
def get_json(url, timeout=10):
request = urllib.request.Request(url, headers={"Accept": "application/json"})
with urllib.request.urlopen(request, timeout=timeout) as reply:
return json.loads(reply.read().decode("utf-8"))
# ---------- Prometheus ----------
def prom_query(expr, at=None):
"""Мгновенный запрос. Возвращает список (labels, float). Ошибка запроса -> исключение."""
params = {"query": expr}
if at is not None:
params["time"] = str(at)
data = get_json(f"{PROMETHEUS_URL}/api/v1/query?" + urllib.parse.urlencode(params))
if data.get("status") != "success":
raise RuntimeError(f"PromQL error: {data}")
result = data["data"]["result"]
if data["data"]["resultType"] == "scalar":
return [({}, float(result[1]))]
return [(item["metric"], float(item["value"][1])) for item in result]
def prom_value(expr, default=None):
rows = prom_query(expr)
return rows[0][1] if rows else default
def prom_series(match):
data = get_json(f"{PROMETHEUS_URL}/api/v1/series?" + urllib.parse.urlencode({"match[]": match}))
return data["data"]
def alerts():
"""Активные алерты: {имя: состояние (pending/firing)}."""
data = get_json(f"{PROMETHEUS_URL}/api/v1/alerts")
out = {}
for alert in data["data"]["alerts"]:
name = alert["labels"]["alertname"]
if out.get(name) != "firing":
out[name] = alert["state"]
return out
def wait_alert(name, state, seconds, poll=5, log=print):
"""Ждёт, пока алерт name перейдёт в state ('firing' или 'resolved' = отсутствует). Возвращает секунды."""
started = time.monotonic()
while True:
current = alerts().get(name, "inactive")
done = current == "firing" if state == "firing" else current == "inactive"
if done:
return round(time.monotonic() - started, 1)
if time.monotonic() - started > seconds:
raise AssertionError(f"alert {name}: still {current} after {seconds} s (want {state})")
log(f" ... {name}: {current} ({time.monotonic() - started:.0f} s)")
time.sleep(poll)
def wait_metric(expr, predicate, seconds, poll=5):
"""Ждёт, пока значение выражения удовлетворит predicate. Возвращает значение."""
deadline = time.monotonic() + seconds
while True:
value = prom_value(expr)
if value is not None and predicate(value):
return value
if time.monotonic() > deadline:
raise AssertionError(f"{expr}: value {value} after {seconds} s")
time.sleep(poll)
# ---------- Loki ----------
def loki_query(logql, minutes=30, limit=200):
"""Записи за последние minutes минут: список (labels+structured metadata, строка)."""
now = time.time_ns()
params = {"query": logql, "start": str(now - minutes * 60 * 10**9), "end": str(now), "limit": str(limit),
"direction": "backward"}
data = get_json(f"{LOKI_URL}/loki/api/v1/query_range?" + urllib.parse.urlencode(params))
if data.get("status") != "success":
raise RuntimeError(f"LogQL error: {data}")
out = []
for stream in data["data"]["result"]:
for value in stream["values"]:
labels = dict(stream["stream"])
if len(value) > 2 and isinstance(value[2], dict):
for group in value[2].values():
if isinstance(group, dict):
labels.update(group)
out.append((labels, value[1]))
return out
def wait_logs(logql, minimum=1, seconds=60):
deadline = time.monotonic() + seconds
while True:
rows = loki_query(logql)
if len(rows) >= minimum:
return rows
if time.monotonic() > deadline:
raise AssertionError(f"Loki: {len(rows)} records for {logql} after {seconds} s")
time.sleep(2)
# ---------- Tempo ----------
def new_traceparent():
"""Свой trace_id для контролируемого запроса: по нему трасса находится без поиска."""
trace_id = secrets.token_hex(16)
return trace_id, f"00-{trace_id}-{secrets.token_hex(8)}-01"
def tempo_trace(trace_id):
"""Спаны трассы: список dict(service, name, kind, span_id, parent_id, duration_ms, status, attrs, links)."""
try:
data = get_json(f"{TEMPO_URL}/api/traces/{trace_id}")
except urllib.error.HTTPError as error:
if error.code == 404:
return []
raise
spans = []
for batch in data.get("batches", data.get("resourceSpans", [])):
resource = {a["key"]: _value(a["value"]) for a in batch["resource"].get("attributes", [])}
for scope in batch.get("scopeSpans", batch.get("instrumentationLibrarySpans", [])):
for span in scope.get("spans", []):
spans.append({
"service": resource.get("service.name"),
"instance": resource.get("service.instance.id"),
"name": span["name"],
"kind": span.get("kind", "").replace("SPAN_KIND_", ""),
"span_id": _hex_id(span["spanId"], 16),
"parent_id": _hex_id(span.get("parentSpanId", ""), 16),
"start": int(span["startTimeUnixNano"]),
"duration_ms": (int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"])) / 1e6,
"status": span.get("status", {}).get("code", "").replace("STATUS_CODE_", ""),
"attrs": {a["key"]: _value(a["value"]) for a in span.get("attributes", [])},
"links": [{"trace_id": _hex_id(link.get("traceId", ""), 32),
"span_id": _hex_id(link.get("spanId", ""), 16)} for link in span.get("links", [])],
})
return spans
def _hex_id(value, size):
"""ID спана/трассы из JSON Tempo: hex или base64 (OTLP JSON кодирует bytes в base64) -> hex."""
if not value:
return ""
if re.fullmatch(r"[0-9a-fA-F]+", value) and len(value) in (size, size // 2 * 2):
return value.lower().rjust(size, "0")
try:
return base64.b64decode(value).hex().rjust(size, "0")
except (binascii.Error, ValueError):
return value
def _value(value):
for key in ("stringValue", "intValue", "boolValue", "doubleValue"):
if key in value:
return int(value[key]) if key == "intValue" else value[key]
return json.dumps(value)
def wait_trace(trace_id, predicate=lambda spans: bool(spans), seconds=60):
deadline = time.monotonic() + seconds
while True:
spans = tempo_trace(trace_id)
if predicate(spans):
return spans
if time.monotonic() > deadline:
raise AssertionError(f"Tempo: trace {trace_id} incomplete after {seconds} s ({len(spans)} spans)")
time.sleep(2)
def tempo_search(traceql, minutes=30, limit=20):
now = int(time.time())
params = {"q": traceql, "start": str(now - minutes * 60), "end": str(now), "limit": str(limit)}
return get_json(f"{TEMPO_URL}/api/search?" + urllib.parse.urlencode(params)).get("traces", [])
def span_tree(spans):
"""Текстовое дерево трассы для отчёта (без атрибутов с данными пользователя)."""
by_id = {s["span_id"]: s for s in spans}
children = {}
for span in spans:
parent = span["parent_id"] if span["parent_id"] in by_id else ""
children.setdefault(parent, []).append(span)
lines = []
def walk(span, depth):
extra = []
for key in ("messenger.job.attempt", "messenger.job.outcome", "http.response.status_code", "error.type",
"aws.s3.key"):
if key in span["attrs"]:
extra.append(f"{key}={span['attrs'][key]}")
status = " ERROR" if span["status"] == "ERROR" else ""
links = f" links={len(span['links'])}" if span["links"] else ""
short = (span["service"] or "?").replace("messenger-", "")
lines.append(f"{' ' * depth}- [{short}] {span['name']} ({span['kind']}, {span['duration_ms']:.1f} ms)"
f"{status}{links} {' '.join(extra)}".rstrip())
for child in sorted(children.get(span["span_id"], []), key=lambda s: s["start"]):
walk(child, depth + 1)
for root in sorted(children.get("", []), key=lambda s: s["start"]):
walk(root, 0)
return "\n".join(lines)