Files
mtusi2026lab7/scripts/build_dashboard.py

266 lines
19 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Генерирует infra/grafana/dashboards/messenger.json (импортируемый dashboard, provisioning из Git).
python scripts/build_dashboard.py
Все запросы панелей — здесь, в одном месте. student-tests/check_dashboard.py выполняет каждый запрос
против живых Prometheus/Loki/Tempo и падает, если имя метрики/метки не существует или запрос с ошибкой.
Единицы: секунды (s) для длительностей, reqps/ops для частот, percentunit для долей, bytes для памяти.
"""
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "infra/grafana/dashboards/messenger.json"
PROM = {"type": "prometheus", "uid": "prometheus"}
LOKI = {"type": "loki", "uid": "loki"}
TEMPO = {"type": "tempo", "uid": "tempo"}
API = 'job="messenger-api", http_route!~"/health/.*"' # бизнес-запросы API (знаменатель SLO)
QUEUE = 'messaging_destination_name="attachments.process.v1"'
_panels = []
_next_id = [1]
_y = [0]
def _id():
_next_id[0] += 1
return _next_id[0]
def row(title):
_panels.append({"type": "row", "title": title, "id": _id(), "collapsed": False,
"gridPos": {"h": 1, "w": 24, "x": 0, "y": _y[0]}, "panels": []})
_y[0] += 1
class Line:
"""Раскладка панелей в строку сетки Grafana (24 колонки)."""
def __init__(self, height):
self.x, self.h = 0, height
def place(self, width):
pos = {"h": self.h, "w": width, "x": self.x, "y": _y[0]}
self.x += width
return pos
def end(self):
_y[0] += self.h
def prom(expr, legend="", instant=False, fmt=None):
target = {"datasource": PROM, "expr": expr, "legendFormat": legend or "__auto", "refId": "",
"range": not instant, "instant": instant}
if fmt:
target["format"] = fmt
return target
def _with_refs(targets):
for n, target in enumerate(targets):
target["refId"] = chr(ord("A") + n)
return targets
def timeseries(line, width, title, description, targets, unit, thresholds=None, stack=False, minimum=None):
defaults = {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10, "showPoints": "never",
"stacking": {"mode": "normal" if stack else "none"}}}
if minimum is not None:
defaults["min"] = minimum
if thresholds:
defaults["thresholds"] = {"mode": "absolute", "steps": [{"color": "green", "value": None}] + [
{"color": color, "value": value} for value, color in thresholds]}
defaults["custom"]["thresholdsStyle"] = {"mode": "line+area"}
_panels.append({"type": "timeseries", "id": _id(), "title": title, "description": description,
"datasource": PROM, "gridPos": line.place(width), "targets": _with_refs(targets),
"fieldConfig": {"defaults": defaults, "overrides": []},
"options": {"legend": {"displayMode": "list", "placement": "bottom"},
"tooltip": {"mode": "multi", "sort": "desc"}}})
def stat(line, width, title, description, expr, unit, steps, decimals=None):
defaults = {"unit": unit, "thresholds": {"mode": "absolute", "steps": [
{"color": color, "value": value} for value, color in steps]}, "color": {"mode": "thresholds"}}
if decimals is not None:
defaults["decimals"] = decimals
_panels.append({"type": "stat", "id": _id(), "title": title, "description": description, "datasource": PROM,
"gridPos": line.place(width), "targets": _with_refs([prom(expr, instant=True)]),
"fieldConfig": {"defaults": defaults, "overrides": []},
"options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
"colorMode": "background", "graphMode": "none", "textMode": "value"}})
def quantiles(metric, selector, by="le"):
return [prom(f"histogram_quantile({q}, sum by ({by}) (rate({metric}_bucket{{{selector}}}[$__rate_interval])))",
f"p{int(q * 100)}") for q in (0.5, 0.95, 0.99)]
def build():
row("Обзор и SLO (окно SLO — 1 час)")
line = Line(4)
stat(line, 3, "RPS API", "Бизнес-запросы API в секунду (без /health), все реплики.",
"messenger:http_requests:rate1m", "reqps", [(None, "blue")])
stat(line, 3, "Доля 5xx (1m)", "5xx / все бизнес-запросы за 1 минуту. Порог алерта 5%.",
"messenger:http_error_ratio:rate1m", "percentunit", [(None, "green"), (0.01, "orange"), (0.05, "red")], 2)
stat(line, 3, "p95 API (5m)", "95-й перцентиль из buckets гистограммы всех реплик (не среднее p95).",
"messenger:http_request_duration_seconds:p95_5m", "s", [(None, "green"), (0.25, "orange"), (1, "red")])
stat(line, 3, "Старейшее задание", "Возраст старейшего готового к обработке задания (queue lag). Порог алерта 30 с.",
f"max(messenger_queue_oldest_age_seconds{{{QUEUE}}})", "s", [(None, "green"), (10, "orange"), (30, "red")])
stat(line, 3, "SLO-1 доступность", "Доля бизнес-запросов без 5xx за 1 час. Цель 99%.",
"messenger:slo_availability:ratio_1h", "percentunit", [(None, "red"), (0.99, "green")], 2)
stat(line, 3, "Бюджет ошибок SLO-1", "Остаток бюджета ошибок (1% запросов) за 1 час; < 0 — SLO нарушен.",
"messenger:slo_availability:error_budget_remaining_1h", "percentunit",
[(None, "red"), (0, "orange"), (0.5, "green")], 0)
stat(line, 3, "SLO-2 POST < 250 мс", "Доля успешных POST /chats/{chat_id}/messages быстрее 250 мс за 1 час. Цель 95%.",
"messenger:slo_latency_messages:ratio_1h", "percentunit", [(None, "red"), (0.95, "green")], 2)
stat(line, 3, "SLO-3 ready ≤ 30 с", "Доля вложений, ставших ready не позже 30 с после 202, за 1 час. Цель 95%.",
"messenger:slo_time_to_ready:ratio_1h", "percentunit", [(None, "red"), (0.95, "green")], 2)
line.end()
line = Line(5)
_panels.append({"type": "table", "id": _id(), "title": "Алерты Prometheus (pending/firing)",
"description": "ALERTS из Prometheus: имя, состояние, severity. Пусто — все правила inactive.",
"datasource": PROM, "gridPos": line.place(24),
"targets": _with_refs([prom('ALERTS{alertstate=~"pending|firing"}', instant=True, fmt="table")]),
"transformations": [{"id": "organize", "options": {"excludeByName": {
"Time": True, "Value": True, "__name__": True}}}],
"fieldConfig": {"defaults": {}, "overrides": []}, "options": {"showHeader": True}})
line.end()
row("API: запросы, ошибки, задержка (RED)")
line = Line(8)
timeseries(line, 8, "RPS по маршрутам", "Шаблон маршрута, а не URL: метка ограниченной кардинальности.",
[prom(f"sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))",
"{{http_request_method}} {{http_route}}")], "reqps")
timeseries(line, 8, "RPS по классу статуса", "2xx/4xx/5xx. 4xx — ошибки клиента, в SLO не входят.",
[prom(f"sum by (http_response_status_class) (rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))",
"{{http_response_status_class}}")], "reqps", stack=True)
timeseries(line, 8, "Доля 5xx", "Числитель: 5xx бизнес-запросов; знаменатель: все бизнес-запросы. Линия — порог алерта.",
[prom(f'(sum(rate(http_server_request_duration_seconds_count{{{API}, http_response_status_class="5xx"}}[$__rate_interval])) or on() vector(0)) '
f"/ sum(rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))", "5xx ratio")],
"percentunit", thresholds=[(0.05, "red")], minimum=0)
line.end()
line = Line(8)
timeseries(line, 8, "Задержка API p50 / p95 / p99", "histogram_quantile по сумме buckets всех реплик.",
quantiles("http_server_request_duration_seconds", API), "s")
timeseries(line, 8, "p95 по маршрутам", "Где именно медленно.",
[prom(f"histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{{{API}}}[$__rate_interval])))",
"{{http_route}}")], "s")
timeseries(line, 8, "Запросы в обработке (по репликам)", "http.server.active_requests: насыщение реплик API.",
[prom('sum by (instance) (http_server_active_requests{job="messenger-api"})', "{{instance}}")], "short")
line.end()
row("Фоновая обработка: очередь, worker, время до ready")
line = Line(8)
timeseries(line, 8, "Попытки заданий в секунду по исходу", "ready / retry / failed / stale / lease_lost (messenger.job.duration count).",
[prom("sum by (outcome) (rate(messenger_job_duration_seconds_count[$__rate_interval]))", "{{outcome}}")],
"ops", stack=True)
timeseries(line, 8, "Длительность попытки p50 / p95 / p99 (ready)", "От claim до COMMIT ready в worker.",
quantiles("messenger_job_duration_seconds", 'outcome="ready"'), "s")
timeseries(line, 8, "Время до ready p50 / p95 / p99", "От устойчивого приёма (COMMIT задания в API) до COMMIT ready. Линия — SLO-3 (30 с).",
quantiles("messenger_attachment_time_to_ready_seconds", 'job_type="attachment.process.v1"'), "s",
thresholds=[(30, "red")])
line.end()
line = Line(8)
timeseries(line, 8, "Глубина очереди и backlog", "Сообщения в RabbitMQ (очередь и DLQ) и незавершённые задания в PostgreSQL.",
[prom("max by (messaging_destination_name) (messenger_queue_depth)", "broker {{messaging_destination_name}}"),
prom("max by (status) (messenger_jobs_backlog)", "jobs {{status}}")], "short")
timeseries(line, 8, "Возраст старейшего задания и outbox", "Queue lag (worker не успевает) и publish lag (dispatcher/брокер). Линия — порог алерта.",
[prom(f"max(messenger_queue_oldest_age_seconds{{{QUEUE}}})", "oldest queued job"),
prom("max(messenger_outbox_oldest_age_seconds)", "oldest unpublished outbox row")], "s",
thresholds=[(30, "red")])
timeseries(line, 8, "Доставки и публикации", "Worker: processed / noop / dead_lettered / requeued; dispatcher: ok / error.",
[prom("sum by (outcome) (rate(messenger_messages_consumed_total[$__rate_interval]))", "consumed {{outcome}}"),
prom("sum by (outcome) (rate(messenger_messages_published_total[$__rate_interval]))", "published {{outcome}}")],
"ops")
line.end()
row("Насыщение: пул БД, worker, CPU/RAM")
line = Line(8)
timeseries(line, 8, "Пул PostgreSQL API: занято / максимум / ждут", "used растёт до max и появляются ждущие — запросы ждут соединение (медленная БД или блокировки).",
[prom('sum by (instance) (db_client_connection_count{db_client_connection_state="used"})', "used {{instance}}"),
prom("max(db_client_connection_max)", "max per replica"),
prom("sum by (instance) (db_client_connection_pending_requests)", "waiting {{instance}}")], "short")
timeseries(line, 8, "Ожидание соединения из пула p95", "Время получения соединения из пула API (не время запроса).",
[prom("histogram_quantile(0.95, sum by (le, instance) (rate(db_client_connection_wait_time_seconds_bucket[$__rate_interval])))",
"p95 {{instance}}")], "s")
timeseries(line, 8, "Worker: занятость (busy / capacity)", "Доля занятых потоков-потребителей. 100% — worker насыщен, очередь растёт.",
[prom("sum(messenger_worker_busy) / sum(messenger_worker_capacity)", "saturation")], "percentunit",
thresholds=[(0.8, "orange"), (1, "red")], minimum=0)
line.end()
line = Line(8)
timeseries(line, 8, "CPU процессов (ядра)", "rate(process.cpu.time): ядер на процесс.",
[prom("sum by (job, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))", "{{job}} {{instance}}")],
"short")
timeseries(line, 8, "CPU контейнеров / лимит", "cgroup v2 (только в Docker): доля лимита CPU контейнера, включая дочерние процессы worker.",
[prom("sum by (job, instance) (rate(container_cpu_time_seconds_total[$__rate_interval])) / on (job, instance) max by (job, instance) (container_cpu_limit)",
"{{job}} {{instance}}")], "percentunit")
timeseries(line, 8, "Память: RSS процесса и контейнер", "process.memory.usage (RSS) и container.memory.usage (cgroup).",
[prom("max by (job, instance) (process_memory_usage_bytes)", "rss {{job}} {{instance}}"),
prom("max by (job, instance) (container_memory_usage_bytes)", "cgroup {{job}} {{instance}}")], "bytes")
line.end()
row("Конвейер телеметрии: Collector и SDK")
line = Line(8)
timeseries(line, 8, "Collector: принято и отправлено", "Спаны/записи логов/точки метрик в секунду.",
[prom("sum(rate(otelcol_receiver_accepted_spans[$__rate_interval]))", "accepted spans"),
prom("sum(rate(otelcol_receiver_accepted_log_records[$__rate_interval]))", "accepted logs"),
prom("sum(rate(otelcol_receiver_accepted_metric_points[$__rate_interval]))", "accepted metric points"),
prom("sum by (exporter) (rate(otelcol_exporter_sent_spans[$__rate_interval]))", "sent spans {{exporter}}"),
prom("sum by (exporter) (rate(otelcol_exporter_sent_log_records[$__rate_interval]))", "sent logs {{exporter}}")],
"ops")
timeseries(line, 8, "Collector: потери и отказы", "send_failed — истёк повтор; enqueue_failed — очередь полна; refused — memory_limiter.",
[prom("sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval]))", "send failed spans {{exporter}}"),
prom("sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))", "send failed logs {{exporter}}"),
prom("sum by (exporter) (rate(otelcol_exporter_enqueue_failed_spans[$__rate_interval]))", "queue full spans {{exporter}}"),
prom("sum(rate(otelcol_receiver_refused_spans[$__rate_interval]))", "refused spans (memory_limiter)")],
"ops")
timeseries(line, 8, "Очереди exporter и RSS Collector", "Заполнение sending_queue (доля ёмкости) и память процесса Collector.",
[prom("max by (exporter) (otelcol_exporter_queue_size / otelcol_exporter_queue_capacity)", "queue {{exporter}}")],
"percentunit", minimum=0)
line.end()
line = Line(8)
timeseries(line, 12, "Collector RSS", "Ограничено memory_limiter (200 MiB) при недоступном хранилище.",
[prom("max(otelcol_process_memory_rss)", "collector rss")], "bytes")
timeseries(line, 12, "SDK приложений: экспорт и потери", "Потери в процессе приложения: processor error.type=queue_full (очередь 2048 полна), exporter error.type (Collector не принял пакет).",
[prom('sum by (job) (rate(otel_sdk_exporter_span_exported_total{error_type=""}[$__rate_interval]))', "exported spans {{job}}"),
prom('sum by (job, error_type) (rate(otel_sdk_exporter_span_exported_total{error_type!=""}[$__rate_interval]))', "export failed spans {{job}} {{error_type}}"),
prom('sum by (job, error_type) (rate(otel_sdk_exporter_log_exported_total{error_type!=""}[$__rate_interval]))', "export failed logs {{job}} {{error_type}}"),
prom('sum by (job) (rate(otel_sdk_processor_span_processed_total{error_type!=""}[$__rate_interval]))', "queue full spans {{job}}")],
"ops")
line.end()
row("Логи и трассы")
line = Line(9)
_panels.append({"type": "logs", "id": _id(), "title": "Предупреждения и ошибки (Loki)",
"description": "Структурные логи всех сервисов; trace_id в записи ведёт в Tempo.",
"datasource": LOKI, "gridPos": line.place(12),
"targets": _with_refs([{"datasource": LOKI, "refId": "", "queryType": "range",
"expr": '{service_name=~"messenger-.*"} | detected_level=~"warn|error"'}]),
"options": {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending",
"enableLogDetails": True}})
_panels.append({"type": "table", "id": _id(), "title": "Трассы с ошибками (Tempo, TraceQL)",
"description": "Последние трассы со статусом ERROR: 5xx API, неудачные попытки заданий.",
"datasource": TEMPO, "gridPos": line.place(12),
"targets": _with_refs([{"datasource": TEMPO, "refId": "", "queryType": "traceql",
"query": "{ status = error }", "limit": 20, "tableType": "traces"}]),
"fieldConfig": {"defaults": {}, "overrides": []}, "options": {"showHeader": True}})
line.end()
return {
"uid": "messenger-lab6", "title": "Messenger — lab6 observability", "tags": ["messenger", "lab6"],
"timezone": "browser", "editable": False, "graphTooltip": 1, "schemaVersion": 39, "version": 1,
"refresh": "10s", "time": {"from": "now-30m", "to": "now"},
"templating": {"list": []}, "annotations": {"list": [{
"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, "enable": True, "hide": True,
"iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard"}]},
"panels": _panels,
}
if __name__ == "__main__":
OUT.parent.mkdir(parents=True, exist_ok=True)
OUT.write_text(json.dumps(build(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(f"written {OUT.relative_to(ROOT)}: {len(_panels)} panels")