"""Генерирует infra/grafana/dashboards/messenger.json (импортируемый dashboard, provisioning из Git). python scripts/build_dashboard.py Все запросы панелей — здесь, в одном месте. student-tests/check_dashboard.py выполняет каждый запрос против живых Prometheus/Loki/Tempo и падает, если имя метрики/метки не существует или запрос с ошибкой. Единицы: секунды (s) для длительностей, reqps/ops для частот, percentunit для долей, bytes для памяти. """ import json from pathlib import Path ROOT = Path(__file__).resolve().parents[1] OUT = ROOT / "infra/grafana/dashboards/messenger.json" PROM = {"type": "prometheus", "uid": "prometheus"} LOKI = {"type": "loki", "uid": "loki"} TEMPO = {"type": "tempo", "uid": "tempo"} API = 'job="messenger-api", http_route!~"/health/.*"' # бизнес-запросы API (знаменатель SLO) QUEUE = 'messaging_destination_name="attachments.process.v1"' _panels = [] _next_id = [1] _y = [0] def _id(): _next_id[0] += 1 return _next_id[0] def row(title): _panels.append({"type": "row", "title": title, "id": _id(), "collapsed": False, "gridPos": {"h": 1, "w": 24, "x": 0, "y": _y[0]}, "panels": []}) _y[0] += 1 class Line: """Раскладка панелей в строку сетки Grafana (24 колонки).""" def __init__(self, height): self.x, self.h = 0, height def place(self, width): pos = {"h": self.h, "w": width, "x": self.x, "y": _y[0]} self.x += width return pos def end(self): _y[0] += self.h def prom(expr, legend="", instant=False, fmt=None): target = {"datasource": PROM, "expr": expr, "legendFormat": legend or "__auto", "refId": "", "range": not instant, "instant": instant} if fmt: target["format"] = fmt return target def _with_refs(targets): for n, target in enumerate(targets): target["refId"] = chr(ord("A") + n) return targets def timeseries(line, width, title, description, targets, unit, thresholds=None, stack=False, minimum=None): defaults = {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "stacking": {"mode": "normal" if stack else "none"}}} if minimum is not None: defaults["min"] = minimum if thresholds: defaults["thresholds"] = {"mode": "absolute", "steps": [{"color": "green", "value": None}] + [ {"color": color, "value": value} for value, color in thresholds]} defaults["custom"]["thresholdsStyle"] = {"mode": "line+area"} _panels.append({"type": "timeseries", "id": _id(), "title": title, "description": description, "datasource": PROM, "gridPos": line.place(width), "targets": _with_refs(targets), "fieldConfig": {"defaults": defaults, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi", "sort": "desc"}}}) def stat(line, width, title, description, expr, unit, steps, decimals=None): defaults = {"unit": unit, "thresholds": {"mode": "absolute", "steps": [ {"color": color, "value": value} for value, color in steps]}, "color": {"mode": "thresholds"}} if decimals is not None: defaults["decimals"] = decimals _panels.append({"type": "stat", "id": _id(), "title": title, "description": description, "datasource": PROM, "gridPos": line.place(width), "targets": _with_refs([prom(expr, instant=True)]), "fieldConfig": {"defaults": defaults, "overrides": []}, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, "colorMode": "background", "graphMode": "none", "textMode": "value"}}) def quantiles(metric, selector, by="le"): return [prom(f"histogram_quantile({q}, sum by ({by}) (rate({metric}_bucket{{{selector}}}[$__rate_interval])))", f"p{int(q * 100)}") for q in (0.5, 0.95, 0.99)] def build(): row("Обзор и SLO (окно SLO — 1 час)") line = Line(4) stat(line, 3, "RPS API", "Бизнес-запросы API в секунду (без /health), все реплики.", "messenger:http_requests:rate1m", "reqps", [(None, "blue")]) stat(line, 3, "Доля 5xx (1m)", "5xx / все бизнес-запросы за 1 минуту. Порог алерта 5%.", "messenger:http_error_ratio:rate1m", "percentunit", [(None, "green"), (0.01, "orange"), (0.05, "red")], 2) stat(line, 3, "p95 API (5m)", "95-й перцентиль из buckets гистограммы всех реплик (не среднее p95).", "messenger:http_request_duration_seconds:p95_5m", "s", [(None, "green"), (0.25, "orange"), (1, "red")]) stat(line, 3, "Старейшее задание", "Возраст старейшего готового к обработке задания (queue lag). Порог алерта 30 с.", f"max(messenger_queue_oldest_age_seconds{{{QUEUE}}})", "s", [(None, "green"), (10, "orange"), (30, "red")]) stat(line, 3, "SLO-1 доступность", "Доля бизнес-запросов без 5xx за 1 час. Цель 99%.", "messenger:slo_availability:ratio_1h", "percentunit", [(None, "red"), (0.99, "green")], 2) stat(line, 3, "Бюджет ошибок SLO-1", "Остаток бюджета ошибок (1% запросов) за 1 час; < 0 — SLO нарушен.", "messenger:slo_availability:error_budget_remaining_1h", "percentunit", [(None, "red"), (0, "orange"), (0.5, "green")], 0) stat(line, 3, "SLO-2 POST < 250 мс", "Доля успешных POST /chats/{chat_id}/messages быстрее 250 мс за 1 час. Цель 95%.", "messenger:slo_latency_messages:ratio_1h", "percentunit", [(None, "red"), (0.95, "green")], 2) stat(line, 3, "SLO-3 ready ≤ 30 с", "Доля вложений, ставших ready не позже 30 с после 202, за 1 час. Цель 95%.", "messenger:slo_time_to_ready:ratio_1h", "percentunit", [(None, "red"), (0.95, "green")], 2) line.end() line = Line(5) _panels.append({"type": "table", "id": _id(), "title": "Алерты Prometheus (pending/firing)", "description": "ALERTS из Prometheus: имя, состояние, severity. Пусто — все правила inactive.", "datasource": PROM, "gridPos": line.place(24), "targets": _with_refs([prom('ALERTS{alertstate=~"pending|firing"}', instant=True, fmt="table")]), "transformations": [{"id": "organize", "options": {"excludeByName": { "Time": True, "Value": True, "__name__": True}}}], "fieldConfig": {"defaults": {}, "overrides": []}, "options": {"showHeader": True}}) line.end() row("API: запросы, ошибки, задержка (RED)") line = Line(8) timeseries(line, 8, "RPS по маршрутам", "Шаблон маршрута, а не URL: метка ограниченной кардинальности.", [prom(f"sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))", "{{http_request_method}} {{http_route}}")], "reqps") timeseries(line, 8, "RPS по классу статуса", "2xx/4xx/5xx. 4xx — ошибки клиента, в SLO не входят.", [prom(f"sum by (http_response_status_class) (rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))", "{{http_response_status_class}}")], "reqps", stack=True) timeseries(line, 8, "Доля 5xx", "Числитель: 5xx бизнес-запросов; знаменатель: все бизнес-запросы. Линия — порог алерта.", [prom(f'(sum(rate(http_server_request_duration_seconds_count{{{API}, http_response_status_class="5xx"}}[$__rate_interval])) or on() vector(0)) ' f"/ sum(rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))", "5xx ratio")], "percentunit", thresholds=[(0.05, "red")], minimum=0) line.end() line = Line(8) timeseries(line, 8, "Задержка API p50 / p95 / p99", "histogram_quantile по сумме buckets всех реплик.", quantiles("http_server_request_duration_seconds", API), "s") timeseries(line, 8, "p95 по маршрутам", "Где именно медленно.", [prom(f"histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{{{API}}}[$__rate_interval])))", "{{http_route}}")], "s") timeseries(line, 8, "Запросы в обработке (по репликам)", "http.server.active_requests: насыщение реплик API.", [prom('sum by (instance) (http_server_active_requests{job="messenger-api"})', "{{instance}}")], "short") line.end() row("Фоновая обработка: очередь, worker, время до ready") line = Line(8) timeseries(line, 8, "Попытки заданий в секунду по исходу", "ready / retry / failed / stale / lease_lost (messenger.job.duration count).", [prom("sum by (outcome) (rate(messenger_job_duration_seconds_count[$__rate_interval]))", "{{outcome}}")], "ops", stack=True) timeseries(line, 8, "Длительность попытки p50 / p95 / p99 (ready)", "От claim до COMMIT ready в worker.", quantiles("messenger_job_duration_seconds", 'outcome="ready"'), "s") timeseries(line, 8, "Время до ready p50 / p95 / p99", "От устойчивого приёма (COMMIT задания в API) до COMMIT ready. Линия — SLO-3 (30 с).", quantiles("messenger_attachment_time_to_ready_seconds", 'job_type="attachment.process.v1"'), "s", thresholds=[(30, "red")]) line.end() line = Line(8) timeseries(line, 8, "Глубина очереди и backlog", "Сообщения в RabbitMQ (очередь и DLQ) и незавершённые задания в PostgreSQL.", [prom("max by (messaging_destination_name) (messenger_queue_depth)", "broker {{messaging_destination_name}}"), prom("max by (status) (messenger_jobs_backlog)", "jobs {{status}}")], "short") timeseries(line, 8, "Возраст старейшего задания и outbox", "Queue lag (worker не успевает) и publish lag (dispatcher/брокер). Линия — порог алерта.", [prom(f"max(messenger_queue_oldest_age_seconds{{{QUEUE}}})", "oldest queued job"), prom("max(messenger_outbox_oldest_age_seconds)", "oldest unpublished outbox row")], "s", thresholds=[(30, "red")]) timeseries(line, 8, "Доставки и публикации", "Worker: processed / noop / dead_lettered / requeued; dispatcher: ok / error.", [prom("sum by (outcome) (rate(messenger_messages_consumed_total[$__rate_interval]))", "consumed {{outcome}}"), prom("sum by (outcome) (rate(messenger_messages_published_total[$__rate_interval]))", "published {{outcome}}")], "ops") line.end() row("Насыщение: пул БД, worker, CPU/RAM") line = Line(8) timeseries(line, 8, "Пул PostgreSQL API: занято / максимум / ждут", "used растёт до max и появляются ждущие — запросы ждут соединение (медленная БД или блокировки).", [prom('sum by (instance) (db_client_connection_count{db_client_connection_state="used"})', "used {{instance}}"), prom("max(db_client_connection_max)", "max per replica"), prom("sum by (instance) (db_client_connection_pending_requests)", "waiting {{instance}}")], "short") timeseries(line, 8, "Ожидание соединения из пула p95", "Время получения соединения из пула API (не время запроса).", [prom("histogram_quantile(0.95, sum by (le, instance) (rate(db_client_connection_wait_time_seconds_bucket[$__rate_interval])))", "p95 {{instance}}")], "s") timeseries(line, 8, "Worker: занятость (busy / capacity)", "Доля занятых потоков-потребителей. 100% — worker насыщен, очередь растёт.", [prom("sum(messenger_worker_busy) / sum(messenger_worker_capacity)", "saturation")], "percentunit", thresholds=[(0.8, "orange"), (1, "red")], minimum=0) line.end() line = Line(8) timeseries(line, 8, "CPU процессов (ядра)", "rate(process.cpu.time): ядер на процесс.", [prom("sum by (job, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))", "{{job}} {{instance}}")], "short") timeseries(line, 8, "CPU контейнеров / лимит", "cgroup v2 (только в Docker): доля лимита CPU контейнера, включая дочерние процессы worker.", [prom("sum by (job, instance) (rate(container_cpu_time_seconds_total[$__rate_interval])) / on (job, instance) max by (job, instance) (container_cpu_limit)", "{{job}} {{instance}}")], "percentunit") timeseries(line, 8, "Память: RSS процесса и контейнер", "process.memory.usage (RSS) и container.memory.usage (cgroup).", [prom("max by (job, instance) (process_memory_usage_bytes)", "rss {{job}} {{instance}}"), prom("max by (job, instance) (container_memory_usage_bytes)", "cgroup {{job}} {{instance}}")], "bytes") line.end() row("Конвейер телеметрии: Collector и SDK") line = Line(8) timeseries(line, 8, "Collector: принято и отправлено", "Спаны/записи логов/точки метрик в секунду.", [prom("sum(rate(otelcol_receiver_accepted_spans[$__rate_interval]))", "accepted spans"), prom("sum(rate(otelcol_receiver_accepted_log_records[$__rate_interval]))", "accepted logs"), prom("sum(rate(otelcol_receiver_accepted_metric_points[$__rate_interval]))", "accepted metric points"), prom("sum by (exporter) (rate(otelcol_exporter_sent_spans[$__rate_interval]))", "sent spans {{exporter}}"), prom("sum by (exporter) (rate(otelcol_exporter_sent_log_records[$__rate_interval]))", "sent logs {{exporter}}")], "ops") timeseries(line, 8, "Collector: потери и отказы", "send_failed — истёк повтор; enqueue_failed — очередь полна; refused — memory_limiter.", [prom("sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval]))", "send failed spans {{exporter}}"), prom("sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))", "send failed logs {{exporter}}"), prom("sum by (exporter) (rate(otelcol_exporter_enqueue_failed_spans[$__rate_interval]))", "queue full spans {{exporter}}"), prom("sum(rate(otelcol_receiver_refused_spans[$__rate_interval]))", "refused spans (memory_limiter)")], "ops") timeseries(line, 8, "Очереди exporter и RSS Collector", "Заполнение sending_queue (доля ёмкости) и память процесса Collector.", [prom("max by (exporter) (otelcol_exporter_queue_size / otelcol_exporter_queue_capacity)", "queue {{exporter}}")], "percentunit", minimum=0) line.end() line = Line(8) timeseries(line, 12, "Collector RSS", "Ограничено memory_limiter (200 MiB) при недоступном хранилище.", [prom("max(otelcol_process_memory_rss)", "collector rss")], "bytes") timeseries(line, 12, "SDK приложений: экспорт и потери", "Потери в процессе приложения: processor error.type=queue_full (очередь 2048 полна), exporter error.type (Collector не принял пакет).", [prom('sum by (job) (rate(otel_sdk_exporter_span_exported_total{error_type=""}[$__rate_interval]))', "exported spans {{job}}"), prom('sum by (job, error_type) (rate(otel_sdk_exporter_span_exported_total{error_type!=""}[$__rate_interval]))', "export failed spans {{job}} {{error_type}}"), prom('sum by (job, error_type) (rate(otel_sdk_exporter_log_exported_total{error_type!=""}[$__rate_interval]))', "export failed logs {{job}} {{error_type}}"), prom('sum by (job) (rate(otel_sdk_processor_span_processed_total{error_type!=""}[$__rate_interval]))', "queue full spans {{job}}")], "ops") line.end() row("Логи и трассы") line = Line(9) _panels.append({"type": "logs", "id": _id(), "title": "Предупреждения и ошибки (Loki)", "description": "Структурные логи всех сервисов; trace_id в записи ведёт в Tempo.", "datasource": LOKI, "gridPos": line.place(12), "targets": _with_refs([{"datasource": LOKI, "refId": "", "queryType": "range", "expr": '{service_name=~"messenger-.*"} | detected_level=~"warn|error"'}]), "options": {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True}}) _panels.append({"type": "table", "id": _id(), "title": "Трассы с ошибками (Tempo, TraceQL)", "description": "Последние трассы со статусом ERROR: 5xx API, неудачные попытки заданий.", "datasource": TEMPO, "gridPos": line.place(12), "targets": _with_refs([{"datasource": TEMPO, "refId": "", "queryType": "traceql", "query": "{ status = error }", "limit": 20, "tableType": "traces"}]), "fieldConfig": {"defaults": {}, "overrides": []}, "options": {"showHeader": True}}) line.end() return { "uid": "messenger-lab6", "title": "Messenger — lab6 observability", "tags": ["messenger", "lab6"], "timezone": "browser", "editable": False, "graphTooltip": 1, "schemaVersion": 39, "version": 1, "refresh": "10s", "time": {"from": "now-30m", "to": "now"}, "templating": {"list": []}, "annotations": {"list": [{ "builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, "enable": True, "hide": True, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard"}]}, "panels": _panels, } if __name__ == "__main__": OUT.parent.mkdir(parents=True, exist_ok=True) OUT.write_text(json.dumps(build(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print(f"written {OUT.relative_to(ROOT)}: {len(_panels)} panels")