Files
mtusi2026lab6/student-tests/obs.py
T
2026-11-01 20:38:50 +03:00

211 lines
9.2 KiB
Python

"""lab6: запросы к хранилищам телеметрии для собственных проверок (только стандартная библиотека).
- Prometheus (метрики и alerts): http://127.0.0.1:9090 — опубликован только на 127.0.0.1;
- Loki (логи) и Tempo (трассы) наружу не опубликованы: запросы идут через Grafana
(http://127.0.0.1:3000, прокси источника данных /api/datasources/proxy/uid/<uid>/...).
Пароль администратора Grafana читается из secrets/grafana_admin_password и не печатается.
Адреса можно переопределить: PROM_URL, GRAFANA_URL, LOKI_URL, TEMPO_URL (прямой адрес без Grafana).
"""
import base64
import json
import os
import time
import urllib.error
import urllib.parse
import urllib.request
from helpers import ROOT, check
PROM_URL = os.environ.get("PROM_URL", "http://127.0.0.1:9090").rstrip("/")
GRAFANA_URL = os.environ.get("GRAFANA_URL", "http://127.0.0.1:3000").rstrip("/")
LOKI_URL = os.environ.get("LOKI_URL", "").rstrip("/")
TEMPO_URL = os.environ.get("TEMPO_URL", "").rstrip("/")
def _grafana_auth() -> str:
path = ROOT / "secrets" / "grafana_admin_password"
check(path.exists(), "secrets/grafana_admin_password not found: run sh scripts/init-secrets.sh")
password = path.read_text(encoding="utf-8").strip()
return "Basic " + base64.b64encode(f"admin:{password}".encode()).decode()
def get_json(url: str, auth: str | None = None, timeout: float = 15.0):
request = urllib.request.Request(url, headers={"Authorization": auth} if auth else {})
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
return json.loads(response.read() or b"null")
except urllib.error.HTTPError as exc:
body = exc.read()[:300].decode("utf-8", "replace")
raise RuntimeError(f"HTTP {exc.code} from {url.split('?')[0]}: {body}") from None
def grafana(path: str):
return get_json(GRAFANA_URL + path, _grafana_auth())
def _via(direct: str, uid: str) -> tuple[str, str | None]:
if direct:
return direct, None
return f"{GRAFANA_URL}/api/datasources/proxy/uid/{uid}", _grafana_auth()
# ---------- Prometheus ----------
def prom(query: str, at: float | None = None) -> list[dict]:
"""Instant query (сейчас или на момент at, unix-время) -> [{'metric': {...}, 'value': float}]."""
params = {"query": query} | ({"time": f"{at:.3f}"} if at else {})
data = get_json(PROM_URL + "/api/v1/query?" + urllib.parse.urlencode(params))
check(data["status"] == "success", f"prometheus: {data}")
result = data["data"]["result"]
if data["data"]["resultType"] == "scalar":
return [{"metric": {}, "value": float(result[1])}]
return [{"metric": r["metric"], "value": float(r["value"][1])} for r in result]
def prom_value(query: str, default: float | None = None, at: float | None = None) -> float | None:
rows = prom(query, at)
return rows[0]["value"] if rows else default
def alerts() -> dict:
"""{alertname: 'pending'|'firing'} — активные alerts Prometheus."""
data = get_json(PROM_URL + "/api/v1/alerts")
return {a["labels"]["alertname"]: a["state"] for a in data["data"]["alerts"]}
def rules() -> list[dict]:
data = get_json(PROM_URL + "/api/v1/rules")
return [r for g in data["data"]["groups"] for r in g["rules"]]
# ---------- Loki ----------
def loki(query: str, minutes: int = 15, limit: int = 200) -> list[dict]:
"""Записи логов: [{'labels': {...}, 'line': str, 'ts': int}] (labels включают structured metadata)."""
base, auth = _via(LOKI_URL, "loki")
end = time.time_ns()
params = {"query": query, "limit": limit, "start": end - minutes * 60 * 10**9, "end": end, "direction": "backward"}
data = get_json(base + "/loki/api/v1/query_range?" + urllib.parse.urlencode(params), auth)
out = []
for stream in data["data"]["result"]:
for ts, line in stream["values"]:
out.append({"labels": stream["stream"], "line": line, "ts": int(ts)})
return sorted(out, key=lambda e: e["ts"])
def wait_logs(query: str, count: int = 1, seconds: float = 60) -> list[dict]:
deadline = time.monotonic() + seconds
while True:
rows = loki(query)
if len(rows) >= count:
return rows
check(time.monotonic() < deadline, f"loki: no logs for {query}")
time.sleep(2)
# ---------- Tempo ----------
def tempo_search(traceql: str, limit: int = 20, minutes: int = 60) -> list[dict]:
base, auth = _via(TEMPO_URL, "tempo")
now = int(time.time())
params = {"q": traceql, "limit": limit, "start": now - minutes * 60, "end": now + 60}
data = get_json(base + "/api/search?" + urllib.parse.urlencode(params), auth)
return data.get("traces", [])
def tempo_trace(trace_id: str) -> list[dict]:
"""Все спаны трассы: [{'service', 'name', 'kind', 'span_id', 'parent', 'start', 'end', 'status', 'attrs'}]."""
base, auth = _via(TEMPO_URL, "tempo")
data = get_json(f"{base}/api/traces/{trace_id}", auth)
spans = []
for batch in data.get("batches", data.get("resourceSpans", [])):
resource = {a["key"]: _value(a["value"]) for a in batch["resource"].get("attributes", [])}
for scope in batch.get("scopeSpans", []):
for s in scope["spans"]:
spans.append({
"service": resource.get("service.name"), "instance": resource.get("service.instance.id"),
"name": s["name"], "kind": s.get("kind", "").replace("SPAN_KIND_", ""),
"span_id": _hex(s["spanId"]), "parent": _hex(s.get("parentSpanId", "")),
"start": int(s["startTimeUnixNano"]), "end": int(s["endTimeUnixNano"]),
"status": s.get("status", {}).get("code", "").replace("STATUS_CODE_", ""),
"attrs": {a["key"]: _value(a["value"]) for a in s.get("attributes", [])},
})
return sorted(spans, key=lambda sp: sp["start"])
def wait_trace(traceql: str, predicate=lambda spans: True, seconds: float = 60) -> tuple[str, list[dict]]:
"""Дождаться трассы по TraceQL, у которой выполняется predicate(спаны). Возвращает (trace_id, спаны)."""
deadline = time.monotonic() + seconds
while True:
for found in tempo_search(traceql):
spans = tempo_trace(found["traceID"])
if spans and predicate(spans):
return _hex(found["traceID"]), spans
check(time.monotonic() < deadline, f"tempo: no complete trace for {traceql}")
time.sleep(3)
def _hex(value: str) -> str:
"""Tempo отдаёт ID в hex (/api/traces) или base64 (часть версий): приводим к hex."""
if not value:
return ""
if all(c in "0123456789abcdefABCDEF" for c in value) and len(value) in (16, 32):
return value.lower()
try:
return base64.b64decode(value).hex()
except ValueError:
return value
def _value(v: dict):
if "intValue" in v:
return int(v["intValue"]) # в JSON Tempo int64 приходит строкой
for key in ("stringValue", "doubleValue", "boolValue"):
if key in v:
return v[key]
return json.dumps(v)
def tree(spans: list[dict]) -> list[str]:
"""Дерево трассы для evidence: отступ = вложенность, длительность в мс."""
by_id = {s["span_id"]: s for s in spans}
children = {}
for s in spans:
parent = s["parent"] if s["parent"] in by_id else ""
children.setdefault(parent, []).append(s)
lines = []
def walk(span, depth):
keys = ("http.route", "http.response.status_code", "messenger.job.attempt", "messenger.job.outcome",
"messenger.job.error_code", "rpc.method")
extra = " ".join(f"{k.split('.')[-1]}={span['attrs'][k]}" for k in keys if k in span["attrs"])
name = span["name"] if span["kind"] != "CLIENT" or "rpc.method" in span["attrs"] \
else f"SQL {span['name']}"
lines.append(f"{' ' * depth}{span['service']:<17}{span['kind']:<9}{name[:48]:<48}"
f"{(span['end'] - span['start']) / 1e6:9.1f} ms {span['status'] or ''} {extra}".rstrip())
for child in children.get(span["span_id"], []):
walk(child, depth + 1)
for root in children.get("", []):
walk(root, 0)
return lines
def main(argv):
if argv[:1] == ["alerts"]:
for rule in rules():
if rule.get("type") == "alerting":
print(f"{rule['name']:<24} {rule['state']:<9} for {rule.get('duration', 0):>4.0f} s health={rule['health']}")
elif argv[:1] == ["logs"]:
for entry in loki('{service_name=~"messenger-api|messenger-worker"} | severity_text=~"WARN.*|ERROR"', minutes=30, limit=30):
labels = entry["labels"]
print(f"{labels.get('service_name')} {labels.get('severity_text')} {labels.get('event')} "
f"trace_id={labels.get('trace_id', '-')} {entry['line'][:120]}")
else:
print("usage: python student-tests/obs.py alerts|logs")
if __name__ == "__main__":
import sys
main(sys.argv[1:])