Files
mtusi2026lab7/student-tests/check_dashboard.py

120 lines
6.0 KiB
Python

"""Lab6: каждый запрос dashboard и правил выполняется против живых хранилищ.
python student-tests/check_dashboard.py
Проверяет, что dashboard ссылается на реально существующие метрики/метки (контракт наблюдаемости):
- каждое PromQL-выражение панелей и recording/alert rules выполняется без ошибки;
- для ключевых сигналов (RED API, задания, очередь, пул, worker, CPU/RAM) результат НЕ пуст;
- LogQL логов и TraceQL трасс выполняются;
- в метках метрик приложения нет запрещённых (user_id, chat_id, message_id, attachment_id, request_id,
url, filename, text) и нет значений-UUID (кроме instance).
Перед запуском нужен трафик: make test (smoke) — он создаёт запросы, задание и логи.
"""
import json
import re
import sys
from pathlib import Path
import obs
from common import check
ROOT = Path(__file__).resolve().parents[1]
DASHBOARD = ROOT / "infra/grafana/dashboards/messenger.json"
RULES = ROOT / "infra/prometheus/rules"
FORBIDDEN_LABELS = {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url", "url_full",
"filename", "text", "job_id", "trace_id", "span_id", "authorization"}
UUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
# Метрики, которые обязаны иметь данные после smoke (имя -> зачем)
MUST_HAVE_DATA = {
'http_server_request_duration_seconds_count{job="messenger-api"}': "HTTP requests/duration (3)",
"messenger_job_duration_seconds_count": "jobs processed/failed + duration (3)",
"messenger_attachment_time_to_ready_seconds_count": "end-to-end time to ready (4)",
"messenger_queue_depth": "queue depth (4)",
"messenger_queue_oldest_age_seconds": "oldest job age (4)",
"db_client_connection_count": "DB pool used/idle (4)",
"db_client_connection_pending_requests": "DB pool waiting (4)",
"messenger_worker_busy": "worker busy (4)",
"messenger_worker_capacity": "worker capacity (4)",
"process_cpu_time_seconds_total": "process CPU (4)",
"process_memory_usage_bytes": "process RAM (4)",
"otelcol_exporter_queue_size": "collector queue (4)",
}
def substitute(expr):
return expr.replace("$__rate_interval", "1m").replace("$__interval", "1m").replace("$__range", "1h")
def main():
dashboard = json.loads(DASHBOARD.read_text(encoding="utf-8"))
prom_exprs, loki_exprs, tempo_exprs = [], [], []
for panel in dashboard["panels"]:
for target in panel.get("targets", []):
uid = target.get("datasource", {}).get("uid")
if uid == "prometheus":
prom_exprs.append((panel["title"], target["expr"]))
elif uid == "loki":
loki_exprs.append((panel["title"], target["expr"]))
elif uid == "tempo":
tempo_exprs.append((panel["title"], target["query"]))
try:
import yaml # noqa: F401 — если PyYAML есть, проверим и выражения правил
rules = []
for path in sorted(RULES.glob("*.yml")):
for group in yaml.safe_load(path.read_text(encoding="utf-8"))["groups"]:
for rule in group["rules"]:
rules.append((f"{path.name}: {rule.get('record') or rule.get('alert')}", rule["expr"]))
except ImportError:
rules = []
empty, errors = [], []
for title, expr in prom_exprs + rules:
try:
rows = obs.prom_query(substitute(expr))
except Exception as exc: # noqa: BLE001
errors.append(f"{title}: {exc}")
continue
if not rows:
empty.append(title)
print(f"PromQL: {len(prom_exprs)} dashboard expressions + {len(rules)} rule expressions executed, "
f"{len(errors)} errors, {len(empty)} empty now (no traffic of that kind / Docker-only / no incident)")
for title in sorted(set(empty)):
print(f" empty now: {title}")
check(not errors, "PromQL errors:\n" + "\n".join(errors))
for title, expr in loki_exprs:
obs.loki_query(expr, minutes=60, limit=5)
print(f"LogQL ok: {title}: {expr}")
for title, query in tempo_exprs:
found = obs.tempo_search(query, minutes=60, limit=5)
print(f"TraceQL ok: {title}: {query} -> {len(found)} trace(s)")
print()
for expr, why in MUST_HAVE_DATA.items():
rows = obs.prom_query(expr)
check(rows, f"no data for {expr} ({why})")
print(f"data: {expr:<70} {len(rows):>3} series <- {why}")
print()
bad = []
for job in ("messenger-api", "messenger-worker", "messenger-dispatcher", "messenger-janitor"):
for series in obs.prom_series(f'{{job="{job}"}}'):
for name, value in series.items():
if name in FORBIDDEN_LABELS:
bad.append(f"{series['__name__']}: forbidden label {name}")
if name != "instance" and UUID.search(value):
bad.append(f"{series['__name__']}: UUID value in label {name}")
labels = sorted({name for job in ("messenger-api", "messenger-worker", "messenger-dispatcher")
for series in obs.prom_series(f'{{job="{job}"}}') for name in series})
print(f"labels used by application metrics: {', '.join(labels)}")
check(not bad, "high-cardinality labels:\n" + "\n".join(sorted(set(bad))[:20]))
route_values = sorted({row[0].get("http_route") for row in obs.prom_query(
'count by (http_route) (http_server_request_duration_seconds_count{job="messenger-api"})')})
print(f"http_route values (templates, not URLs): {route_values}")
check(all(not UUID.search(value or "") for value in route_values), "raw URL in http_route")
print("PASS: every dashboard/rule query runs, key signals have data, no forbidden labels")
if __name__ == "__main__":
sys.exit(main())