"""Lab6: каждый запрос dashboard и правил выполняется против живых хранилищ. python student-tests/check_dashboard.py Проверяет, что dashboard ссылается на реально существующие метрики/метки (контракт наблюдаемости): - каждое PromQL-выражение панелей и recording/alert rules выполняется без ошибки; - для ключевых сигналов (RED API, задания, очередь, пул, worker, CPU/RAM) результат НЕ пуст; - LogQL логов и TraceQL трасс выполняются; - в метках метрик приложения нет запрещённых (user_id, chat_id, message_id, attachment_id, request_id, url, filename, text) и нет значений-UUID (кроме instance). Перед запуском нужен трафик: make test (smoke) — он создаёт запросы, задание и логи. """ import json import re import sys from pathlib import Path import obs from common import check ROOT = Path(__file__).resolve().parents[1] DASHBOARD = ROOT / "infra/grafana/dashboards/messenger.json" RULES = ROOT / "infra/prometheus/rules" FORBIDDEN_LABELS = {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url", "url_full", "filename", "text", "job_id", "trace_id", "span_id", "authorization"} UUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") # Метрики, которые обязаны иметь данные после smoke (имя -> зачем) MUST_HAVE_DATA = { 'http_server_request_duration_seconds_count{job="messenger-api"}': "HTTP requests/duration (3)", "messenger_job_duration_seconds_count": "jobs processed/failed + duration (3)", "messenger_attachment_time_to_ready_seconds_count": "end-to-end time to ready (4)", "messenger_queue_depth": "queue depth (4)", "messenger_queue_oldest_age_seconds": "oldest job age (4)", "db_client_connection_count": "DB pool used/idle (4)", "db_client_connection_pending_requests": "DB pool waiting (4)", "messenger_worker_busy": "worker busy (4)", "messenger_worker_capacity": "worker capacity (4)", "process_cpu_time_seconds_total": "process CPU (4)", "process_memory_usage_bytes": "process RAM (4)", "otelcol_exporter_queue_size": "collector queue (4)", } def substitute(expr): return expr.replace("$__rate_interval", "1m").replace("$__interval", "1m").replace("$__range", "1h") def main(): dashboard = json.loads(DASHBOARD.read_text(encoding="utf-8")) prom_exprs, loki_exprs, tempo_exprs = [], [], [] for panel in dashboard["panels"]: for target in panel.get("targets", []): uid = target.get("datasource", {}).get("uid") if uid == "prometheus": prom_exprs.append((panel["title"], target["expr"])) elif uid == "loki": loki_exprs.append((panel["title"], target["expr"])) elif uid == "tempo": tempo_exprs.append((panel["title"], target["query"])) try: import yaml # noqa: F401 — если PyYAML есть, проверим и выражения правил rules = [] for path in sorted(RULES.glob("*.yml")): for group in yaml.safe_load(path.read_text(encoding="utf-8"))["groups"]: for rule in group["rules"]: rules.append((f"{path.name}: {rule.get('record') or rule.get('alert')}", rule["expr"])) except ImportError: rules = [] empty, errors = [], [] for title, expr in prom_exprs + rules: try: rows = obs.prom_query(substitute(expr)) except Exception as exc: # noqa: BLE001 errors.append(f"{title}: {exc}") continue if not rows: empty.append(title) print(f"PromQL: {len(prom_exprs)} dashboard expressions + {len(rules)} rule expressions executed, " f"{len(errors)} errors, {len(empty)} empty now (no traffic of that kind / Docker-only / no incident)") for title in sorted(set(empty)): print(f" empty now: {title}") check(not errors, "PromQL errors:\n" + "\n".join(errors)) for title, expr in loki_exprs: obs.loki_query(expr, minutes=60, limit=5) print(f"LogQL ok: {title}: {expr}") for title, query in tempo_exprs: found = obs.tempo_search(query, minutes=60, limit=5) print(f"TraceQL ok: {title}: {query} -> {len(found)} trace(s)") print() for expr, why in MUST_HAVE_DATA.items(): rows = obs.prom_query(expr) check(rows, f"no data for {expr} ({why})") print(f"data: {expr:<70} {len(rows):>3} series <- {why}") print() bad = [] for job in ("messenger-api", "messenger-worker", "messenger-dispatcher", "messenger-janitor"): for series in obs.prom_series(f'{{job="{job}"}}'): for name, value in series.items(): if name in FORBIDDEN_LABELS: bad.append(f"{series['__name__']}: forbidden label {name}") if name != "instance" and UUID.search(value): bad.append(f"{series['__name__']}: UUID value in label {name}") labels = sorted({name for job in ("messenger-api", "messenger-worker", "messenger-dispatcher") for series in obs.prom_series(f'{{job="{job}"}}') for name in series}) print(f"labels used by application metrics: {', '.join(labels)}") check(not bad, "high-cardinality labels:\n" + "\n".join(sorted(set(bad))[:20])) route_values = sorted({row[0].get("http_route") for row in obs.prom_query( 'count by (http_route) (http_server_request_duration_seconds_count{job="messenger-api"})')}) print(f"http_route values (templates, not URLs): {route_values}") check(all(not UUID.search(value or "") for value in route_values), "raw URL in http_route") print("PASS: every dashboard/rule query runs, key signals have data, no forbidden labels") if __name__ == "__main__": sys.exit(main())