"""Запросы к хранилищам телеметрии для тестов lab6 (только стандартная библиотека). Адреса — порты, опубликованные compose.yaml только на 127.0.0.1: PROMETHEUS_URL (http://127.0.0.1:9090), LOKI_URL (http://127.0.0.1:3100), TEMPO_URL (http://127.0.0.1:3200). """ import base64 import binascii import json import os import re import secrets import time import urllib.error import urllib.parse import urllib.request PROMETHEUS_URL = os.environ.get("PROMETHEUS_URL", "http://127.0.0.1:9090").rstrip("/") LOKI_URL = os.environ.get("LOKI_URL", "http://127.0.0.1:3100").rstrip("/") TEMPO_URL = os.environ.get("TEMPO_URL", "http://127.0.0.1:3200").rstrip("/") def get_json(url, timeout=10): request = urllib.request.Request(url, headers={"Accept": "application/json"}) with urllib.request.urlopen(request, timeout=timeout) as reply: return json.loads(reply.read().decode("utf-8")) # ---------- Prometheus ---------- def prom_query(expr, at=None): """Мгновенный запрос. Возвращает список (labels, float). Ошибка запроса -> исключение.""" params = {"query": expr} if at is not None: params["time"] = str(at) data = get_json(f"{PROMETHEUS_URL}/api/v1/query?" + urllib.parse.urlencode(params)) if data.get("status") != "success": raise RuntimeError(f"PromQL error: {data}") result = data["data"]["result"] if data["data"]["resultType"] == "scalar": return [({}, float(result[1]))] return [(item["metric"], float(item["value"][1])) for item in result] def prom_value(expr, default=None): rows = prom_query(expr) return rows[0][1] if rows else default def prom_series(match): data = get_json(f"{PROMETHEUS_URL}/api/v1/series?" + urllib.parse.urlencode({"match[]": match})) return data["data"] def alerts(): """Активные алерты: {имя: состояние (pending/firing)}.""" data = get_json(f"{PROMETHEUS_URL}/api/v1/alerts") out = {} for alert in data["data"]["alerts"]: name = alert["labels"]["alertname"] if out.get(name) != "firing": out[name] = alert["state"] return out def wait_alert(name, state, seconds, poll=5, log=print): """Ждёт, пока алерт name перейдёт в state ('firing' или 'resolved' = отсутствует). Возвращает секунды.""" started = time.monotonic() while True: current = alerts().get(name, "inactive") done = current == "firing" if state == "firing" else current == "inactive" if done: return round(time.monotonic() - started, 1) if time.monotonic() - started > seconds: raise AssertionError(f"alert {name}: still {current} after {seconds} s (want {state})") log(f" ... {name}: {current} ({time.monotonic() - started:.0f} s)") time.sleep(poll) def wait_metric(expr, predicate, seconds, poll=5): """Ждёт, пока значение выражения удовлетворит predicate. Возвращает значение.""" deadline = time.monotonic() + seconds while True: value = prom_value(expr) if value is not None and predicate(value): return value if time.monotonic() > deadline: raise AssertionError(f"{expr}: value {value} after {seconds} s") time.sleep(poll) # ---------- Loki ---------- def loki_query(logql, minutes=30, limit=200): """Записи за последние minutes минут: список (labels+structured metadata, строка).""" now = time.time_ns() params = {"query": logql, "start": str(now - minutes * 60 * 10**9), "end": str(now), "limit": str(limit), "direction": "backward"} data = get_json(f"{LOKI_URL}/loki/api/v1/query_range?" + urllib.parse.urlencode(params)) if data.get("status") != "success": raise RuntimeError(f"LogQL error: {data}") out = [] for stream in data["data"]["result"]: for value in stream["values"]: labels = dict(stream["stream"]) if len(value) > 2 and isinstance(value[2], dict): for group in value[2].values(): if isinstance(group, dict): labels.update(group) out.append((labels, value[1])) return out def wait_logs(logql, minimum=1, seconds=60): deadline = time.monotonic() + seconds while True: rows = loki_query(logql) if len(rows) >= minimum: return rows if time.monotonic() > deadline: raise AssertionError(f"Loki: {len(rows)} records for {logql} after {seconds} s") time.sleep(2) # ---------- Tempo ---------- def new_traceparent(): """Свой trace_id для контролируемого запроса: по нему трасса находится без поиска.""" trace_id = secrets.token_hex(16) return trace_id, f"00-{trace_id}-{secrets.token_hex(8)}-01" def tempo_trace(trace_id): """Спаны трассы: список dict(service, name, kind, span_id, parent_id, duration_ms, status, attrs, links).""" try: data = get_json(f"{TEMPO_URL}/api/traces/{trace_id}") except urllib.error.HTTPError as error: if error.code == 404: return [] raise spans = [] for batch in data.get("batches", data.get("resourceSpans", [])): resource = {a["key"]: _value(a["value"]) for a in batch["resource"].get("attributes", [])} for scope in batch.get("scopeSpans", batch.get("instrumentationLibrarySpans", [])): for span in scope.get("spans", []): spans.append({ "service": resource.get("service.name"), "instance": resource.get("service.instance.id"), "name": span["name"], "kind": span.get("kind", "").replace("SPAN_KIND_", ""), "span_id": _hex_id(span["spanId"], 16), "parent_id": _hex_id(span.get("parentSpanId", ""), 16), "start": int(span["startTimeUnixNano"]), "duration_ms": (int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"])) / 1e6, "status": span.get("status", {}).get("code", "").replace("STATUS_CODE_", ""), "attrs": {a["key"]: _value(a["value"]) for a in span.get("attributes", [])}, "links": [{"trace_id": _hex_id(link.get("traceId", ""), 32), "span_id": _hex_id(link.get("spanId", ""), 16)} for link in span.get("links", [])], }) return spans def _hex_id(value, size): """ID спана/трассы из JSON Tempo: hex или base64 (OTLP JSON кодирует bytes в base64) -> hex.""" if not value: return "" if re.fullmatch(r"[0-9a-fA-F]+", value) and len(value) in (size, size // 2 * 2): return value.lower().rjust(size, "0") try: return base64.b64decode(value).hex().rjust(size, "0") except (binascii.Error, ValueError): return value def _value(value): for key in ("stringValue", "intValue", "boolValue", "doubleValue"): if key in value: return int(value[key]) if key == "intValue" else value[key] return json.dumps(value) def wait_trace(trace_id, predicate=lambda spans: bool(spans), seconds=60): deadline = time.monotonic() + seconds while True: spans = tempo_trace(trace_id) if predicate(spans): return spans if time.monotonic() > deadline: raise AssertionError(f"Tempo: trace {trace_id} incomplete after {seconds} s ({len(spans)} spans)") time.sleep(2) def tempo_search(traceql, minutes=30, limit=20): now = int(time.time()) params = {"q": traceql, "start": str(now - minutes * 60), "end": str(now), "limit": str(limit)} return get_json(f"{TEMPO_URL}/api/search?" + urllib.parse.urlencode(params)).get("traces", []) def span_tree(spans): """Текстовое дерево трассы для отчёта (без атрибутов с данными пользователя).""" by_id = {s["span_id"]: s for s in spans} children = {} for span in spans: parent = span["parent_id"] if span["parent_id"] in by_id else "" children.setdefault(parent, []).append(span) lines = [] def walk(span, depth): extra = [] for key in ("messenger.job.attempt", "messenger.job.outcome", "http.response.status_code", "error.type", "aws.s3.key"): if key in span["attrs"]: extra.append(f"{key}={span['attrs'][key]}") status = " ERROR" if span["status"] == "ERROR" else "" links = f" links={len(span['links'])}" if span["links"] else "" short = (span["service"] or "?").replace("messenger-", "") lines.append(f"{' ' * depth}- [{short}] {span['name']} ({span['kind']}, {span['duration_ms']:.1f} ms)" f"{status}{links} {' '.join(extra)}".rstrip()) for child in sorted(children.get(span["span_id"], []), key=lambda s: s["start"]): walk(child, depth + 1) for root in sorted(children.get("", []), key=lambda s: s["start"]): walk(root, 0) return "\n".join(lines)