#!/usr/bin/env python3 """Генератор импортируемого dashboard Grafana: infra/grafana/dashboards/messenger.json. JSON лежит в Git и подхватывается provisioning-ом; этот скрипт — воспроизводимый способ его изменить (python3 scripts/build_dashboard.py). Только стандартная библиотека. Все запросы ссылаются на реальные имена метрик (см. REPORT.md «Схема сигналов»). Перцентили — histogram_quantile по sum by (le) buckets всех реплик, не среднее p95. """ import json from pathlib import Path OUT = Path(__file__).resolve().parents[1] / "infra/grafana/dashboards/messenger.json" PROM = {"type": "prometheus", "uid": "prometheus"} LOKI = {"type": "loki", "uid": "loki"} TEMPO = {"type": "tempo", "uid": "tempo"} API = 'job="messenger-api", instance=~"$instance"' BUSINESS = API + ', http_route!~"/health/.*|unmatched"' _panels = [] _id = [0] _pos = {"x": 0, "y": 0, "h": 0} def _next_id(): _id[0] += 1 return _id[0] def place(w, h): if _pos["x"] + w > 24: _pos["x"] = 0 _pos["y"] += _pos["h"] _pos["h"] = 0 g = {"x": _pos["x"], "y": _pos["y"], "w": w, "h": h} _pos["x"] += w _pos["h"] = max(_pos["h"], h) return g def row(title): _pos["x"] = 0 _pos["y"] += _pos["h"] _pos["h"] = 0 _panels.append({"type": "row", "id": _next_id(), "title": title, "collapsed": False, "gridPos": {"x": 0, "y": _pos["y"], "w": 24, "h": 1}, "panels": []}) _pos["y"] += 1 def target(expr, legend="", ref="A", exemplar=False, instant=False): t = {"datasource": PROM, "expr": expr, "legendFormat": legend or "__auto", "refId": ref, "range": not instant, "instant": instant} if exemplar: t["exemplar"] = True return t def ts(title, targets, unit="short", w=8, h=8, description="", stack=False, min0=True, thresholds=None): refs = "ABCDEFGHIJ" tg = [target(*t[:2], ref=refs[i], exemplar=(len(t) > 2 and t[2])) for i, t in enumerate(targets)] p = {"type": "timeseries", "id": _next_id(), "title": title, "description": description, "datasource": PROM, "gridPos": place(w, h), "targets": tg, "fieldConfig": {"defaults": {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "stacking": {"mode": "normal" if stack else "none"}}, **({"min": 0} if min0 else {})}, "overrides": []}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]}, "tooltip": {"mode": "multi", "sort": "desc"}}} if thresholds: p["fieldConfig"]["defaults"]["thresholds"] = {"mode": "absolute", "steps": thresholds} p["fieldConfig"]["defaults"]["custom"]["thresholdsStyle"] = {"mode": "line+area"} _panels.append(p) def stat(title, expr, unit="short", w=3, h=4, steps=None, description="", decimals=None): p = {"type": "stat", "id": _next_id(), "title": title, "description": description, "datasource": PROM, "gridPos": place(w, h), "targets": [target(expr, "", instant=True)], "fieldConfig": {"defaults": {"unit": unit, "thresholds": {"mode": "absolute", "steps": steps or [ {"color": "green", "value": None}]}}, "overrides": []}, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, "colorMode": "background", "graphMode": "area", "textMode": "value"}} if decimals is not None: p["fieldConfig"]["defaults"]["decimals"] = decimals _panels.append(p) def build(): # ------------------------------------------------------------------ обзор row("Обзор и SLO") stat("RPS бизнес-запросов", "messenger:http_requests:rate2m", "reqps", description="sum(rate(http_server_request_duration_seconds_count{job=messenger-api, без /health}[2m]))") stat("5xx, доля (2m)", "messenger:http_error_ratio:rate2m", "percentunit", decimals=2, steps=[{"color": "green", "value": None}, {"color": "orange", "value": 0.01}, {"color": "red", "value": 0.05}], description="Числитель: 5xx бизнес-запросов; знаменатель: все бизнес-запросы (без /health, unmatched).") stat("p95 API (5m)", "messenger:http_latency_seconds:p95_5m", "s", decimals=3, steps=[{"color": "green", "value": None}, {"color": "orange", "value": 0.5}, {"color": "red", "value": 1}]) stat("SLO доступности, 7d", "messenger:slo_availability:ratio_7d", "percentunit", decimals=3, steps=[{"color": "red", "value": None}, {"color": "green", "value": 0.995}], description="SLO: 99.5 % бизнес-запросов без 5xx за скользящие 7 дней.") stat("Остаток error budget, 7d", "messenger:slo_availability:error_budget_remaining_7d", "percentunit", decimals=1, steps=[{"color": "red", "value": None}, {"color": "orange", "value": 0.25}, {"color": "green", "value": 0.5}]) stat("Time to ready p95", "messenger:time_to_ready_seconds:p95_5m", "s", decimals=2, steps=[{"color": "green", "value": None}, {"color": "orange", "value": 10}, {"color": "red", "value": 30}], description="От durable acceptance (COMMIT enqueue) до COMMIT ready. SLO: 99 % не дольше 30 с.") stat("Ready lag очереди", "max(messenger_jobs_ready_lag_seconds) or vector(0)", "s", decimals=0, steps=[{"color": "green", "value": None}, {"color": "orange", "value": 10}, {"color": "red", "value": 30}], description="Сколько ждёт worker самое старое созревшее задание (PostgreSQL jobs).") stat("Alerts firing", 'count(ALERTS{alertstate="firing"}) or vector(0)', "short", steps=[{"color": "green", "value": None}, {"color": "red", "value": 1}]) _panels.append({ "type": "state-timeline", "id": _next_id(), "title": "История alerts (pending/firing -> resolved)", "datasource": PROM, "gridPos": place(24, 6), "targets": [target('max by (alertname, source, alertstate) (ALERTS)', "{{alertname}} {{source}} {{alertstate}}")], "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [ {"color": "transparent", "value": None}, {"color": "red", "value": 1}]}}, "overrides": []}, "options": {"showValue": "never", "mergeValues": True, "rowHeight": 0.8, "legend": {"showLegend": False}}}) # ------------------------------------------------------------------ API row("API: трафик, ошибки, задержка") ts("RPS по маршруту", [("sum by (http_route) (rate(http_server_request_duration_seconds_count{%s}[1m]))" % API, "{{http_route}}")], "reqps", stack=True) ts("Ответы по классу статуса", [( "sum by (http_response_status_class) (rate(http_server_request_duration_seconds_count{%s}[1m]))" % BUSINESS, "{{http_response_status_class}}")], "reqps", stack=True) ts("Доля 5xx: API и LB", [ ("messenger:http_error_ratio:rate2m", "API (без /health)"), ("messenger:lb_error_ratio:rate2m", "HAProxy fe_api")], "percentunit", thresholds=[{"color": "transparent", "value": None}, {"color": "red", "value": 0.05}], description="Если БД недоступна, реплики не ready и 503 отдаёт LB: эти запросы видны только в метрике LB.") ts("Задержка API p50/p95/p99 (из histogram)", [ ("histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p50", True), ("histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p95", True), ("histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p99", True)], "s", w=12, description="Точки-exemplars ведут в трассу Tempo конкретного запроса.") ts("p95 по маршруту", [( "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{%s}[5m])))" % BUSINESS, "{{http_route}}")], "s", w=12) ts("5xx по причине (error.type)", [( 'sum by (error_type, http_route) (rate(http_server_request_duration_seconds_count{%s, http_response_status_class="5xx"}[1m]))' % API, "{{error_type}} {{http_route}}")], "reqps", stack=True) ts("Запросы в обработке (saturation API)", [ ("sum by (instance) (http_server_active_requests{%s})" % API, "{{instance}}")], "short") ts("HAProxy: UP-реплики и очередь LB", [ ('haproxy_backend_active_servers{proxy="be_api"}', "UP реплик"), ('haproxy_backend_current_queue{proxy="be_api"}', "очередь LB")], "short") # ------------------------------------------------------------------ worker row("Worker и очередь заданий") ts("Задания по исходу (в минуту)", [( "sum by (messenger_job_outcome) (increase(messenger_jobs_processed_total[1m]))", "{{messenger_job_outcome}}")], "short", stack=True, description="ready / retry / failed / duplicate / lease_lost / rejected / requeued") ts("Длительность обработки job p50/p95/p99", [ ('histogram_quantile(0.50, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p50", True), ('histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p95", True), ('histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p99", True)], "s") ts("Time to ready p50/p95/p99", [ ('histogram_quantile(0.50, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p50", True), ('histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p95", True), ('histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p99", True)], "s", description="Backlog-ожидание + обработка: время от 202 (COMMIT) до ready.") ts("Глубина очереди: PostgreSQL и RabbitMQ", [ ('max by (messenger_job_status) (messenger_jobs_count{messenger_job_status=~"queued|processing"})', "jobs {{messenger_job_status}} (БД)"), ('max(messenger_outbox_pending)', "outbox не опубликовано"), ('sum(rabbitmq_detailed_queue_messages_ready{queue="attachments.process.v1"})', "RabbitMQ ready"), ('sum(rabbitmq_detailed_queue_messages_unacked{queue="attachments.process.v1"})', "RabbitMQ unacked"), ('sum(rabbitmq_detailed_queue_messages{queue="attachments.dlq"})', "DLQ")], "short") ts("Возраст старейшего задания и lag", [ ('max by (messenger_job_status) (messenger_jobs_oldest_age_seconds)', "oldest {{messenger_job_status}}"), ("max(messenger_jobs_ready_lag_seconds)", "ready lag"), ("max(messenger_outbox_oldest_age_seconds)", "outbox oldest")], "s", thresholds=[{"color": "transparent", "value": None}, {"color": "red", "value": 30}]) ts("Ожидание в очереди p95 (до claim)", [ ("histogram_quantile(0.95, sum by (le) (rate(messenger_job_queue_wait_seconds_bucket[5m])))", "p95 queue wait"), ("histogram_quantile(0.95, sum by (le) (rate(messenger_outbox_publish_delay_seconds_bucket[5m])))", "p95 outbox -> broker")], "s", description="Backlog (сколько ждут) отдельно от processing latency (сколько обрабатываются).") ts("Saturation worker: занятость потребителей", [ ("sum(rate(messenger_worker_busy_time_seconds_total[1m])) / sum(last_over_time(messenger_worker_capacity[30s]))", "utilization (busy_time/capacity)"), ("sum(messenger_worker_busy) / sum(last_over_time(messenger_worker_capacity[30s]))", "busy/capacity (мгновенно)")], "percentunit", description="1.0 = все WORKER_CONCURRENCY потоков всех реплик заняты; очередь растёт быстрее обработки.") ts("Relay: публикации outbox", [ ("sum by (outcome) (rate(messenger_outbox_published_total[1m]))", "{{outcome}}")], "short") # ------------------------------------------------------------------ DB / S3 row("PostgreSQL и S3") ts("Пул соединений API: used / idle / max", [ ('sum by (instance, db_client_connection_state) (db_client_connection_count{%s})' % API, "{{instance}} {{db_client_connection_state}}"), ('max(db_client_connection_max{%s})' % API, "max на реплику")], "short") ts("Ожидающие соединение и таймауты пула", [ ('sum by (instance) (db_client_connection_pending_requests{%s})' % API, "pending {{instance}}"), ('sum by (instance) (increase(db_client_connection_timeouts_total{%s}[1m]))' % API, "timeouts/мин {{instance}}")], "short") ts("Ожидание соединения пула p95", [ ('histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket{%s}[1m])))' % API, "p95 wait")], "s") ts("Операции БД p95 по операции (API)", [ ('histogram_quantile(0.95, sum by (le, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{%s}[1m])))' % API, "{{db_operation_name}}")], "s", w=12) ts("Операции БД worker/relay p95", [ ('histogram_quantile(0.95, sum by (le, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{job=~"messenger-worker|messenger-relay"}[5m])))', "{{db_operation_name}}")], "s", w=12) ts("Ошибки операций БД", [ ('sum by (job, db_operation_name, error_type) (rate(db_client_operation_duration_seconds_count{error_type!=""}[1m]))', "{{job}} {{db_operation_name}} {{error_type}}")], "short") ts("S3 p95 по операции", [ ('histogram_quantile(0.95, sum by (le, messenger_s3_operation) (rate(messenger_s3_operation_duration_seconds_bucket[5m])))', "{{messenger_s3_operation}}")], "s") ts("S3 ошибки", [ ('sum by (job, messenger_s3_operation, outcome) (rate(messenger_s3_operation_duration_seconds_count{outcome!="ok"}[1m]))', "{{job}} {{messenger_s3_operation}} {{outcome}}")], "short") # ------------------------------------------------------------------ ресурсы row("Ресурсы контейнеров (CPU/RAM)") ts("CPU, ядер (cgroup контейнера)", [ ("sum by (job, instance) (rate(container_cpu_time_seconds_total[1m]))", "{{job}} {{instance}}")], "short", w=12) ts("CPU saturation: доля лимита", [ ("rate(container_cpu_time_seconds_total[1m]) / on(job, instance) container_cpu_limit", "{{job}} {{instance}}")], "percentunit", w=12) ts("Память контейнеров приложений", [ ("container_memory_usage_bytes", "{{job}} {{instance}}")], "bytes", w=12) ts("Память: доля лимита", [ ("container_memory_usage_bytes / on(job, instance) container_memory_limit_bytes", "{{job}} {{instance}}")], "percentunit", w=12) ts("Память telemetry-стека (RSS)", [ ('process_resident_memory_bytes{job=~"prometheus|loki|tempo"}', "{{job}}"), ('{__name__=~"otelcol_process_memory_rss(_bytes)?"}', "otel-collector RSS (вкл. страницы бинарника)"), ('otelcol_process_runtime_heap_alloc_bytes', "otel-collector Go heap"), ('rabbitmq_process_resident_memory_bytes', "rabbitmq")], "bytes", w=12) ts("CPU telemetry-стека", [ ('rate(process_cpu_seconds_total{job=~"prometheus|loki|tempo"}[1m])', "{{job}}"), ('rate({__name__=~"otelcol_process_cpu_seconds(_total)?"}[1m])', "otel-collector")], "short", w=12) # ------------------------------------------------------------------ telemetry row("Конвейер телеметрии (потери)") ts("Collector: принято / отправлено / потеряно (spans)", [ ('sum(rate({__name__=~"otelcol_receiver_accepted_spans(_total)?"}[1m]))', "принято"), ('sum(rate({__name__=~"otelcol_exporter_sent_spans(_total)?"}[1m]))', "отправлено в Tempo"), ('sum(rate({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[1m]))', "ошибка отправки"), ('sum(rate({__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?"}[1m]))', "очередь полна (потеря)"), ('sum(rate({__name__=~"otelcol_receiver_refused_spans(_total)?"}[1m]))', "отказ приёма (memory_limiter)")], "short") ts("Collector: логи", [ ('sum(rate({__name__=~"otelcol_receiver_accepted_log_records(_total)?"}[1m]))', "принято"), ('sum(rate({__name__=~"otelcol_exporter_sent_log_records(_total)?"}[1m]))', "отправлено в Loki"), ('sum(rate({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}[1m]))', "ошибка отправки"), ('sum(rate({__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?"}[1m]))', "очередь полна (потеря)")], "short") ts("Collector: очередь экспортёров", [ ("max by (exporter) (otelcol_exporter_queue_size)", "size {{exporter}}"), ("max by (exporter) (otelcol_exporter_queue_capacity)", "capacity {{exporter}}")], "short") ts("SDK приложений: потери на стороне процесса", [ ('sum by (job, error_type) (rate(otel_sdk_processor_span_processed_total{error_type!=""}[1m]))', "spans dropped {{job}} {{error_type}}"), ('sum by (job, error_type) (rate(otel_sdk_processor_log_processed_total{error_type!=""}[1m]))', "logs dropped {{job}} {{error_type}}"), ('sum by (job, error_type) (rate(otel_sdk_exporter_span_exported_total{error_type!=""}[1m]))', "span export failed {{job}} {{error_type}}")], "short", w=12) ts("SDK: заполнение очереди BatchSpanProcessor", [ ("max by (job, instance) (otel_sdk_processor_span_queue_size)", "{{job}} {{instance}}"), ("max(otel_sdk_processor_span_queue_capacity)", "capacity")], "short", w=12) # ------------------------------------------------------------------ logs / traces row("Логи и трассы") _panels.append({ "type": "logs", "id": _next_id(), "title": "Логи WARN/ERROR приложений (Loki)", "datasource": LOKI, "gridPos": place(24, 10), "targets": [{"datasource": LOKI, "refId": "A", "queryType": "range", "expr": '{service_name=~"messenger-.+"} | severity_text=~"WARN.*|ERROR.*|CRITICAL"'}], "options": {"showTime": True, "wrapLogMessage": True, "enableLogDetails": True, "sortOrder": "Descending", "dedupStrategy": "none", "prettifyLogMessage": False}}) _panels.append({ "type": "logs", "id": _next_id(), "title": "HTTP 5xx (event=http.request)", "datasource": LOKI, "gridPos": place(12, 10), "targets": [{"datasource": LOKI, "refId": "A", "queryType": "range", "expr": '{service_name="messenger-api"} | event="http.request" | http_response_status_code >= 500'}], "options": {"showTime": True, "wrapLogMessage": True, "enableLogDetails": True, "sortOrder": "Descending"}}) _panels.append({ "type": "table", "id": _next_id(), "title": "Последние трассы обработки заданий (Tempo)", "datasource": TEMPO, "gridPos": place(12, 10), "targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces", "query": '{ resource.service.name = "messenger-worker" && name = "attachments.process.v1 process" }'}]}) _panels.append({ "type": "table", "id": _next_id(), "title": "Медленные запросы API > 1 s (Tempo)", "datasource": TEMPO, "gridPos": place(12, 10), "targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces", "query": '{ resource.service.name = "messenger-api" && kind = server && duration > 1s }'}]}) _panels.append({ "type": "table", "id": _next_id(), "title": "Трассы с ошибкой (Tempo)", "datasource": TEMPO, "gridPos": place(12, 10), "targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces", "query": '{ status = error }'}]}) return { "uid": "messenger-overview", "title": "Messenger — наблюдаемость (lab6)", "description": "API, worker, очередь, PostgreSQL, S3, ресурсы, конвейер телеметрии. Provisioning из Git.", "tags": ["messenger", "lab6", "otel"], "timezone": "browser", "editable": False, "graphTooltip": 1, "refresh": "10s", "schemaVersion": 39, "version": 1, "time": {"from": "now-30m", "to": "now"}, "templating": {"list": [{ "name": "instance", "label": "API-реплика", "type": "query", "datasource": PROM, "query": {"query": 'label_values(http_server_request_duration_seconds_count{job="messenger-api"}, instance)', "refId": "PrometheusVariableQueryEditor-VariableQuery"}, "definition": 'label_values(http_server_request_duration_seconds_count{job="messenger-api"}, instance)', "includeAll": True, "multi": True, "allValue": ".*", "current": {"text": "All", "value": "$__all"}, "refresh": 2, "sort": 1}]}, "annotations": {"list": [{ "builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, "enable": True, "hide": True, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard"}]}, "panels": _panels, } if __name__ == "__main__": OUT.parent.mkdir(parents=True, exist_ok=True) OUT.write_text(json.dumps(build(), ensure_ascii=False, indent=1) + "\n", encoding="utf-8") print("written %s (%d panels)" % (OUT.relative_to(OUT.parents[3]), len(_panels)))