forked from mtusi2026/mtusi2026lab7
120 lines
6.0 KiB
Python
120 lines
6.0 KiB
Python
"""Lab6: каждый запрос dashboard и правил выполняется против живых хранилищ.
|
|
|
|
python student-tests/check_dashboard.py
|
|
|
|
Проверяет, что dashboard ссылается на реально существующие метрики/метки (контракт наблюдаемости):
|
|
- каждое PromQL-выражение панелей и recording/alert rules выполняется без ошибки;
|
|
- для ключевых сигналов (RED API, задания, очередь, пул, worker, CPU/RAM) результат НЕ пуст;
|
|
- LogQL логов и TraceQL трасс выполняются;
|
|
- в метках метрик приложения нет запрещённых (user_id, chat_id, message_id, attachment_id, request_id,
|
|
url, filename, text) и нет значений-UUID (кроме instance).
|
|
Перед запуском нужен трафик: make test (smoke) — он создаёт запросы, задание и логи.
|
|
"""
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import obs
|
|
from common import check
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DASHBOARD = ROOT / "infra/grafana/dashboards/messenger.json"
|
|
RULES = ROOT / "infra/prometheus/rules"
|
|
FORBIDDEN_LABELS = {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url", "url_full",
|
|
"filename", "text", "job_id", "trace_id", "span_id", "authorization"}
|
|
UUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
|
|
# Метрики, которые обязаны иметь данные после smoke (имя -> зачем)
|
|
MUST_HAVE_DATA = {
|
|
'http_server_request_duration_seconds_count{job="messenger-api"}': "HTTP requests/duration (3)",
|
|
"messenger_job_duration_seconds_count": "jobs processed/failed + duration (3)",
|
|
"messenger_attachment_time_to_ready_seconds_count": "end-to-end time to ready (4)",
|
|
"messenger_queue_depth": "queue depth (4)",
|
|
"messenger_queue_oldest_age_seconds": "oldest job age (4)",
|
|
"db_client_connection_count": "DB pool used/idle (4)",
|
|
"db_client_connection_pending_requests": "DB pool waiting (4)",
|
|
"messenger_worker_busy": "worker busy (4)",
|
|
"messenger_worker_capacity": "worker capacity (4)",
|
|
"process_cpu_time_seconds_total": "process CPU (4)",
|
|
"process_memory_usage_bytes": "process RAM (4)",
|
|
"otelcol_exporter_queue_size": "collector queue (4)",
|
|
}
|
|
|
|
|
|
def substitute(expr):
|
|
return expr.replace("$__rate_interval", "1m").replace("$__interval", "1m").replace("$__range", "1h")
|
|
|
|
|
|
def main():
|
|
dashboard = json.loads(DASHBOARD.read_text(encoding="utf-8"))
|
|
prom_exprs, loki_exprs, tempo_exprs = [], [], []
|
|
for panel in dashboard["panels"]:
|
|
for target in panel.get("targets", []):
|
|
uid = target.get("datasource", {}).get("uid")
|
|
if uid == "prometheus":
|
|
prom_exprs.append((panel["title"], target["expr"]))
|
|
elif uid == "loki":
|
|
loki_exprs.append((panel["title"], target["expr"]))
|
|
elif uid == "tempo":
|
|
tempo_exprs.append((panel["title"], target["query"]))
|
|
try:
|
|
import yaml # noqa: F401 — если PyYAML есть, проверим и выражения правил
|
|
rules = []
|
|
for path in sorted(RULES.glob("*.yml")):
|
|
for group in yaml.safe_load(path.read_text(encoding="utf-8"))["groups"]:
|
|
for rule in group["rules"]:
|
|
rules.append((f"{path.name}: {rule.get('record') or rule.get('alert')}", rule["expr"]))
|
|
except ImportError:
|
|
rules = []
|
|
|
|
empty, errors = [], []
|
|
for title, expr in prom_exprs + rules:
|
|
try:
|
|
rows = obs.prom_query(substitute(expr))
|
|
except Exception as exc: # noqa: BLE001
|
|
errors.append(f"{title}: {exc}")
|
|
continue
|
|
if not rows:
|
|
empty.append(title)
|
|
print(f"PromQL: {len(prom_exprs)} dashboard expressions + {len(rules)} rule expressions executed, "
|
|
f"{len(errors)} errors, {len(empty)} empty now (no traffic of that kind / Docker-only / no incident)")
|
|
for title in sorted(set(empty)):
|
|
print(f" empty now: {title}")
|
|
check(not errors, "PromQL errors:\n" + "\n".join(errors))
|
|
|
|
for title, expr in loki_exprs:
|
|
obs.loki_query(expr, minutes=60, limit=5)
|
|
print(f"LogQL ok: {title}: {expr}")
|
|
for title, query in tempo_exprs:
|
|
found = obs.tempo_search(query, minutes=60, limit=5)
|
|
print(f"TraceQL ok: {title}: {query} -> {len(found)} trace(s)")
|
|
|
|
print()
|
|
for expr, why in MUST_HAVE_DATA.items():
|
|
rows = obs.prom_query(expr)
|
|
check(rows, f"no data for {expr} ({why})")
|
|
print(f"data: {expr:<70} {len(rows):>3} series <- {why}")
|
|
|
|
print()
|
|
bad = []
|
|
for job in ("messenger-api", "messenger-worker", "messenger-dispatcher", "messenger-janitor"):
|
|
for series in obs.prom_series(f'{{job="{job}"}}'):
|
|
for name, value in series.items():
|
|
if name in FORBIDDEN_LABELS:
|
|
bad.append(f"{series['__name__']}: forbidden label {name}")
|
|
if name != "instance" and UUID.search(value):
|
|
bad.append(f"{series['__name__']}: UUID value in label {name}")
|
|
labels = sorted({name for job in ("messenger-api", "messenger-worker", "messenger-dispatcher")
|
|
for series in obs.prom_series(f'{{job="{job}"}}') for name in series})
|
|
print(f"labels used by application metrics: {', '.join(labels)}")
|
|
check(not bad, "high-cardinality labels:\n" + "\n".join(sorted(set(bad))[:20]))
|
|
route_values = sorted({row[0].get("http_route") for row in obs.prom_query(
|
|
'count by (http_route) (http_server_request_duration_seconds_count{job="messenger-api"})')})
|
|
print(f"http_route values (templates, not URLs): {route_values}")
|
|
check(all(not UUID.search(value or "") for value in route_values), "raw URL in http_route")
|
|
print("PASS: every dashboard/rule query runs, key signals have data, no forbidden labels")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|