forked from mtusi2026/mtusi2026lab6
211 lines
9.2 KiB
Python
211 lines
9.2 KiB
Python
"""lab6: запросы к хранилищам телеметрии для собственных проверок (только стандартная библиотека).
|
|
|
|
- Prometheus (метрики и alerts): http://127.0.0.1:9090 — опубликован только на 127.0.0.1;
|
|
- Loki (логи) и Tempo (трассы) наружу не опубликованы: запросы идут через Grafana
|
|
(http://127.0.0.1:3000, прокси источника данных /api/datasources/proxy/uid/<uid>/...).
|
|
Пароль администратора Grafana читается из secrets/grafana_admin_password и не печатается.
|
|
Адреса можно переопределить: PROM_URL, GRAFANA_URL, LOKI_URL, TEMPO_URL (прямой адрес без Grafana).
|
|
"""
|
|
import base64
|
|
import json
|
|
import os
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
from helpers import ROOT, check
|
|
|
|
PROM_URL = os.environ.get("PROM_URL", "http://127.0.0.1:9090").rstrip("/")
|
|
GRAFANA_URL = os.environ.get("GRAFANA_URL", "http://127.0.0.1:3000").rstrip("/")
|
|
LOKI_URL = os.environ.get("LOKI_URL", "").rstrip("/")
|
|
TEMPO_URL = os.environ.get("TEMPO_URL", "").rstrip("/")
|
|
|
|
|
|
def _grafana_auth() -> str:
|
|
path = ROOT / "secrets" / "grafana_admin_password"
|
|
check(path.exists(), "secrets/grafana_admin_password not found: run sh scripts/init-secrets.sh")
|
|
password = path.read_text(encoding="utf-8").strip()
|
|
return "Basic " + base64.b64encode(f"admin:{password}".encode()).decode()
|
|
|
|
|
|
def get_json(url: str, auth: str | None = None, timeout: float = 15.0):
|
|
request = urllib.request.Request(url, headers={"Authorization": auth} if auth else {})
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
return json.loads(response.read() or b"null")
|
|
except urllib.error.HTTPError as exc:
|
|
body = exc.read()[:300].decode("utf-8", "replace")
|
|
raise RuntimeError(f"HTTP {exc.code} from {url.split('?')[0]}: {body}") from None
|
|
|
|
|
|
def grafana(path: str):
|
|
return get_json(GRAFANA_URL + path, _grafana_auth())
|
|
|
|
|
|
def _via(direct: str, uid: str) -> tuple[str, str | None]:
|
|
if direct:
|
|
return direct, None
|
|
return f"{GRAFANA_URL}/api/datasources/proxy/uid/{uid}", _grafana_auth()
|
|
|
|
|
|
# ---------- Prometheus ----------
|
|
|
|
def prom(query: str, at: float | None = None) -> list[dict]:
|
|
"""Instant query (сейчас или на момент at, unix-время) -> [{'metric': {...}, 'value': float}]."""
|
|
params = {"query": query} | ({"time": f"{at:.3f}"} if at else {})
|
|
data = get_json(PROM_URL + "/api/v1/query?" + urllib.parse.urlencode(params))
|
|
check(data["status"] == "success", f"prometheus: {data}")
|
|
result = data["data"]["result"]
|
|
if data["data"]["resultType"] == "scalar":
|
|
return [{"metric": {}, "value": float(result[1])}]
|
|
return [{"metric": r["metric"], "value": float(r["value"][1])} for r in result]
|
|
|
|
|
|
def prom_value(query: str, default: float | None = None, at: float | None = None) -> float | None:
|
|
rows = prom(query, at)
|
|
return rows[0]["value"] if rows else default
|
|
|
|
|
|
def alerts() -> dict:
|
|
"""{alertname: 'pending'|'firing'} — активные alerts Prometheus."""
|
|
data = get_json(PROM_URL + "/api/v1/alerts")
|
|
return {a["labels"]["alertname"]: a["state"] for a in data["data"]["alerts"]}
|
|
|
|
|
|
def rules() -> list[dict]:
|
|
data = get_json(PROM_URL + "/api/v1/rules")
|
|
return [r for g in data["data"]["groups"] for r in g["rules"]]
|
|
|
|
|
|
# ---------- Loki ----------
|
|
|
|
def loki(query: str, minutes: int = 15, limit: int = 200) -> list[dict]:
|
|
"""Записи логов: [{'labels': {...}, 'line': str, 'ts': int}] (labels включают structured metadata)."""
|
|
base, auth = _via(LOKI_URL, "loki")
|
|
end = time.time_ns()
|
|
params = {"query": query, "limit": limit, "start": end - minutes * 60 * 10**9, "end": end, "direction": "backward"}
|
|
data = get_json(base + "/loki/api/v1/query_range?" + urllib.parse.urlencode(params), auth)
|
|
out = []
|
|
for stream in data["data"]["result"]:
|
|
for ts, line in stream["values"]:
|
|
out.append({"labels": stream["stream"], "line": line, "ts": int(ts)})
|
|
return sorted(out, key=lambda e: e["ts"])
|
|
|
|
|
|
def wait_logs(query: str, count: int = 1, seconds: float = 60) -> list[dict]:
|
|
deadline = time.monotonic() + seconds
|
|
while True:
|
|
rows = loki(query)
|
|
if len(rows) >= count:
|
|
return rows
|
|
check(time.monotonic() < deadline, f"loki: no logs for {query}")
|
|
time.sleep(2)
|
|
|
|
|
|
# ---------- Tempo ----------
|
|
|
|
def tempo_search(traceql: str, limit: int = 20, minutes: int = 60) -> list[dict]:
|
|
base, auth = _via(TEMPO_URL, "tempo")
|
|
now = int(time.time())
|
|
params = {"q": traceql, "limit": limit, "start": now - minutes * 60, "end": now + 60}
|
|
data = get_json(base + "/api/search?" + urllib.parse.urlencode(params), auth)
|
|
return data.get("traces", [])
|
|
|
|
|
|
def tempo_trace(trace_id: str) -> list[dict]:
|
|
"""Все спаны трассы: [{'service', 'name', 'kind', 'span_id', 'parent', 'start', 'end', 'status', 'attrs'}]."""
|
|
base, auth = _via(TEMPO_URL, "tempo")
|
|
data = get_json(f"{base}/api/traces/{trace_id}", auth)
|
|
spans = []
|
|
for batch in data.get("batches", data.get("resourceSpans", [])):
|
|
resource = {a["key"]: _value(a["value"]) for a in batch["resource"].get("attributes", [])}
|
|
for scope in batch.get("scopeSpans", []):
|
|
for s in scope["spans"]:
|
|
spans.append({
|
|
"service": resource.get("service.name"), "instance": resource.get("service.instance.id"),
|
|
"name": s["name"], "kind": s.get("kind", "").replace("SPAN_KIND_", ""),
|
|
"span_id": _hex(s["spanId"]), "parent": _hex(s.get("parentSpanId", "")),
|
|
"start": int(s["startTimeUnixNano"]), "end": int(s["endTimeUnixNano"]),
|
|
"status": s.get("status", {}).get("code", "").replace("STATUS_CODE_", ""),
|
|
"attrs": {a["key"]: _value(a["value"]) for a in s.get("attributes", [])},
|
|
})
|
|
return sorted(spans, key=lambda sp: sp["start"])
|
|
|
|
|
|
def wait_trace(traceql: str, predicate=lambda spans: True, seconds: float = 60) -> tuple[str, list[dict]]:
|
|
"""Дождаться трассы по TraceQL, у которой выполняется predicate(спаны). Возвращает (trace_id, спаны)."""
|
|
deadline = time.monotonic() + seconds
|
|
while True:
|
|
for found in tempo_search(traceql):
|
|
spans = tempo_trace(found["traceID"])
|
|
if spans and predicate(spans):
|
|
return _hex(found["traceID"]), spans
|
|
check(time.monotonic() < deadline, f"tempo: no complete trace for {traceql}")
|
|
time.sleep(3)
|
|
|
|
|
|
def _hex(value: str) -> str:
|
|
"""Tempo отдаёт ID в hex (/api/traces) или base64 (часть версий): приводим к hex."""
|
|
if not value:
|
|
return ""
|
|
if all(c in "0123456789abcdefABCDEF" for c in value) and len(value) in (16, 32):
|
|
return value.lower()
|
|
try:
|
|
return base64.b64decode(value).hex()
|
|
except ValueError:
|
|
return value
|
|
|
|
|
|
def _value(v: dict):
|
|
if "intValue" in v:
|
|
return int(v["intValue"]) # в JSON Tempo int64 приходит строкой
|
|
for key in ("stringValue", "doubleValue", "boolValue"):
|
|
if key in v:
|
|
return v[key]
|
|
return json.dumps(v)
|
|
|
|
|
|
def tree(spans: list[dict]) -> list[str]:
|
|
"""Дерево трассы для evidence: отступ = вложенность, длительность в мс."""
|
|
by_id = {s["span_id"]: s for s in spans}
|
|
children = {}
|
|
for s in spans:
|
|
parent = s["parent"] if s["parent"] in by_id else ""
|
|
children.setdefault(parent, []).append(s)
|
|
lines = []
|
|
|
|
def walk(span, depth):
|
|
keys = ("http.route", "http.response.status_code", "messenger.job.attempt", "messenger.job.outcome",
|
|
"messenger.job.error_code", "rpc.method")
|
|
extra = " ".join(f"{k.split('.')[-1]}={span['attrs'][k]}" for k in keys if k in span["attrs"])
|
|
name = span["name"] if span["kind"] != "CLIENT" or "rpc.method" in span["attrs"] \
|
|
else f"SQL {span['name']}"
|
|
lines.append(f"{' ' * depth}{span['service']:<17}{span['kind']:<9}{name[:48]:<48}"
|
|
f"{(span['end'] - span['start']) / 1e6:9.1f} ms {span['status'] or ''} {extra}".rstrip())
|
|
for child in children.get(span["span_id"], []):
|
|
walk(child, depth + 1)
|
|
|
|
for root in children.get("", []):
|
|
walk(root, 0)
|
|
return lines
|
|
|
|
|
|
def main(argv):
|
|
if argv[:1] == ["alerts"]:
|
|
for rule in rules():
|
|
if rule.get("type") == "alerting":
|
|
print(f"{rule['name']:<24} {rule['state']:<9} for {rule.get('duration', 0):>4.0f} s health={rule['health']}")
|
|
elif argv[:1] == ["logs"]:
|
|
for entry in loki('{service_name=~"messenger-api|messenger-worker"} | severity_text=~"WARN.*|ERROR"', minutes=30, limit=30):
|
|
labels = entry["labels"]
|
|
print(f"{labels.get('service_name')} {labels.get('severity_text')} {labels.get('event')} "
|
|
f"trace_id={labels.get('trace_id', '-')} {entry['line'][:120]}")
|
|
else:
|
|
print("usage: python student-tests/obs.py alerts|logs")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
main(sys.argv[1:])
|