Files
mtusi2026lab6/scripts/build_dashboard.py
T
zemiles c199f6632a ЛР 6 — БПИ2303 — Лазарев — на 4
Перенесена реализация lab5 (API x2 за HAProxy, PostgreSQL, SeaweedFS S3,
RabbitMQ, relay outbox, worker x2, janitor) без изменения бизнес-API.

Наблюдаемость:
- OpenTelemetry SDK в API, relay, worker, janitor: traces, metrics, logs
  по OTLP/HTTP; структурные JSON-логи с trace_id/span_id
- OTel Collector (memory_limiter, batch, ограниченные очереди и retry),
  Prometheus (метрики, alert rules), Loki (логи), Tempo (трассы),
  Grafana с provisioning datasources и dashboard
- W3C trace context API -> outbox -> RabbitMQ -> worker (parent-child,
  включая retry); метрики очереди, пула БД, загрузки worker, CPU/RAM
- SLI/SLO, 10 alert rules с unit-тестами promtool и runbooks
- инциденты: остановка worker, блокировка БД, отказ telemetry backend;
  по наблюдениям исправлен простой worker в цикле AMQP (разбор x2.4)
2026-11-17 10:59:41 +03:00

324 lines
23 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Генератор импортируемого dashboard Grafana: infra/grafana/dashboards/messenger.json.
JSON лежит в Git и подхватывается provisioning-ом; этот скрипт — воспроизводимый способ его
изменить (python3 scripts/build_dashboard.py). Только стандартная библиотека.
Все запросы ссылаются на реальные имена метрик (см. REPORT.md «Схема сигналов»).
Перцентили — histogram_quantile по sum by (le) buckets всех реплик, не среднее p95.
"""
import json
from pathlib import Path
OUT = Path(__file__).resolve().parents[1] / "infra/grafana/dashboards/messenger.json"
PROM = {"type": "prometheus", "uid": "prometheus"}
LOKI = {"type": "loki", "uid": "loki"}
TEMPO = {"type": "tempo", "uid": "tempo"}
API = 'job="messenger-api", instance=~"$instance"'
BUSINESS = API + ', http_route!~"/health/.*|unmatched"'
_panels = []
_id = [0]
_pos = {"x": 0, "y": 0, "h": 0}
def _next_id():
_id[0] += 1
return _id[0]
def place(w, h):
if _pos["x"] + w > 24:
_pos["x"] = 0
_pos["y"] += _pos["h"]
_pos["h"] = 0
g = {"x": _pos["x"], "y": _pos["y"], "w": w, "h": h}
_pos["x"] += w
_pos["h"] = max(_pos["h"], h)
return g
def row(title):
_pos["x"] = 0
_pos["y"] += _pos["h"]
_pos["h"] = 0
_panels.append({"type": "row", "id": _next_id(), "title": title, "collapsed": False,
"gridPos": {"x": 0, "y": _pos["y"], "w": 24, "h": 1}, "panels": []})
_pos["y"] += 1
def target(expr, legend="", ref="A", exemplar=False, instant=False):
t = {"datasource": PROM, "expr": expr, "legendFormat": legend or "__auto", "refId": ref,
"range": not instant, "instant": instant}
if exemplar:
t["exemplar"] = True
return t
def ts(title, targets, unit="short", w=8, h=8, description="", stack=False, min0=True, thresholds=None):
refs = "ABCDEFGHIJ"
tg = [target(*t[:2], ref=refs[i], exemplar=(len(t) > 2 and t[2])) for i, t in enumerate(targets)]
p = {"type": "timeseries", "id": _next_id(), "title": title, "description": description,
"datasource": PROM, "gridPos": place(w, h), "targets": tg,
"fieldConfig": {"defaults": {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10,
"showPoints": "never",
"stacking": {"mode": "normal" if stack else "none"}},
**({"min": 0} if min0 else {})},
"overrides": []},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]},
"tooltip": {"mode": "multi", "sort": "desc"}}}
if thresholds:
p["fieldConfig"]["defaults"]["thresholds"] = {"mode": "absolute", "steps": thresholds}
p["fieldConfig"]["defaults"]["custom"]["thresholdsStyle"] = {"mode": "line+area"}
_panels.append(p)
def stat(title, expr, unit="short", w=3, h=4, steps=None, description="", decimals=None):
p = {"type": "stat", "id": _next_id(), "title": title, "description": description, "datasource": PROM,
"gridPos": place(w, h), "targets": [target(expr, "", instant=True)],
"fieldConfig": {"defaults": {"unit": unit, "thresholds": {"mode": "absolute", "steps": steps or [
{"color": "green", "value": None}]}}, "overrides": []},
"options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
"colorMode": "background", "graphMode": "area", "textMode": "value"}}
if decimals is not None:
p["fieldConfig"]["defaults"]["decimals"] = decimals
_panels.append(p)
def build():
# ------------------------------------------------------------------ обзор
row("Обзор и SLO")
stat("RPS бизнес-запросов", "messenger:http_requests:rate2m", "reqps",
description="sum(rate(http_server_request_duration_seconds_count{job=messenger-api, без /health}[2m]))")
stat("5xx, доля (2m)", "messenger:http_error_ratio:rate2m", "percentunit", decimals=2,
steps=[{"color": "green", "value": None}, {"color": "orange", "value": 0.01}, {"color": "red", "value": 0.05}],
description="Числитель: 5xx бизнес-запросов; знаменатель: все бизнес-запросы (без /health, unmatched).")
stat("p95 API (5m)", "messenger:http_latency_seconds:p95_5m", "s", decimals=3,
steps=[{"color": "green", "value": None}, {"color": "orange", "value": 0.5}, {"color": "red", "value": 1}])
stat("SLO доступности, 7d", "messenger:slo_availability:ratio_7d", "percentunit", decimals=3,
steps=[{"color": "red", "value": None}, {"color": "green", "value": 0.995}],
description="SLO: 99.5 % бизнес-запросов без 5xx за скользящие 7 дней.")
stat("Остаток error budget, 7d", "messenger:slo_availability:error_budget_remaining_7d", "percentunit",
decimals=1, steps=[{"color": "red", "value": None}, {"color": "orange", "value": 0.25},
{"color": "green", "value": 0.5}])
stat("Time to ready p95", "messenger:time_to_ready_seconds:p95_5m", "s", decimals=2,
steps=[{"color": "green", "value": None}, {"color": "orange", "value": 10}, {"color": "red", "value": 30}],
description="От durable acceptance (COMMIT enqueue) до COMMIT ready. SLO: 99 % не дольше 30 с.")
stat("Ready lag очереди", "max(messenger_jobs_ready_lag_seconds) or vector(0)", "s", decimals=0,
steps=[{"color": "green", "value": None}, {"color": "orange", "value": 10}, {"color": "red", "value": 30}],
description="Сколько ждёт worker самое старое созревшее задание (PostgreSQL jobs).")
stat("Alerts firing", 'count(ALERTS{alertstate="firing"}) or vector(0)', "short",
steps=[{"color": "green", "value": None}, {"color": "red", "value": 1}])
_panels.append({
"type": "state-timeline", "id": _next_id(), "title": "История alerts (pending/firing -> resolved)",
"datasource": PROM, "gridPos": place(24, 6),
"targets": [target('max by (alertname, source, alertstate) (ALERTS)', "{{alertname}} {{source}} {{alertstate}}")],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"},
"thresholds": {"mode": "absolute", "steps": [
{"color": "transparent", "value": None}, {"color": "red", "value": 1}]}},
"overrides": []},
"options": {"showValue": "never", "mergeValues": True, "rowHeight": 0.8,
"legend": {"showLegend": False}}})
# ------------------------------------------------------------------ API
row("API: трафик, ошибки, задержка")
ts("RPS по маршруту", [("sum by (http_route) (rate(http_server_request_duration_seconds_count{%s}[1m]))" % API,
"{{http_route}}")], "reqps", stack=True)
ts("Ответы по классу статуса", [(
"sum by (http_response_status_class) (rate(http_server_request_duration_seconds_count{%s}[1m]))" % BUSINESS,
"{{http_response_status_class}}")], "reqps", stack=True)
ts("Доля 5xx: API и LB", [
("messenger:http_error_ratio:rate2m", "API (без /health)"),
("messenger:lb_error_ratio:rate2m", "HAProxy fe_api")], "percentunit",
thresholds=[{"color": "transparent", "value": None}, {"color": "red", "value": 0.05}],
description="Если БД недоступна, реплики не ready и 503 отдаёт LB: эти запросы видны только в метрике LB.")
ts("Задержка API p50/p95/p99 (из histogram)", [
("histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p50", True),
("histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p95", True),
("histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p99", True)],
"s", w=12, description="Точки-exemplars ведут в трассу Tempo конкретного запроса.")
ts("p95 по маршруту", [(
"histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{%s}[5m])))" % BUSINESS,
"{{http_route}}")], "s", w=12)
ts("5xx по причине (error.type)", [(
'sum by (error_type, http_route) (rate(http_server_request_duration_seconds_count{%s, http_response_status_class="5xx"}[1m]))' % API,
"{{error_type}} {{http_route}}")], "reqps", stack=True)
ts("Запросы в обработке (saturation API)", [
("sum by (instance) (http_server_active_requests{%s})" % API, "{{instance}}")], "short")
ts("HAProxy: UP-реплики и очередь LB", [
('haproxy_backend_active_servers{proxy="be_api"}', "UP реплик"),
('haproxy_backend_current_queue{proxy="be_api"}', "очередь LB")], "short")
# ------------------------------------------------------------------ worker
row("Worker и очередь заданий")
ts("Задания по исходу (в минуту)", [(
"sum by (messenger_job_outcome) (increase(messenger_jobs_processed_total[1m]))", "{{messenger_job_outcome}}")],
"short", stack=True, description="ready / retry / failed / duplicate / lease_lost / rejected / requeued")
ts("Длительность обработки job p50/p95/p99", [
('histogram_quantile(0.50, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p50", True),
('histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p95", True),
('histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p99", True)],
"s")
ts("Time to ready p50/p95/p99", [
('histogram_quantile(0.50, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p50", True),
('histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p95", True),
('histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p99", True)],
"s", description="Backlog-ожидание + обработка: время от 202 (COMMIT) до ready.")
ts("Глубина очереди: PostgreSQL и RabbitMQ", [
('max by (messenger_job_status) (messenger_jobs_count{messenger_job_status=~"queued|processing"})', "jobs {{messenger_job_status}} (БД)"),
('max(messenger_outbox_pending)', "outbox не опубликовано"),
('sum(rabbitmq_detailed_queue_messages_ready{queue="attachments.process.v1"})', "RabbitMQ ready"),
('sum(rabbitmq_detailed_queue_messages_unacked{queue="attachments.process.v1"})', "RabbitMQ unacked"),
('sum(rabbitmq_detailed_queue_messages{queue="attachments.dlq"})', "DLQ")], "short")
ts("Возраст старейшего задания и lag", [
('max by (messenger_job_status) (messenger_jobs_oldest_age_seconds)', "oldest {{messenger_job_status}}"),
("max(messenger_jobs_ready_lag_seconds)", "ready lag"),
("max(messenger_outbox_oldest_age_seconds)", "outbox oldest")], "s",
thresholds=[{"color": "transparent", "value": None}, {"color": "red", "value": 30}])
ts("Ожидание в очереди p95 (до claim)", [
("histogram_quantile(0.95, sum by (le) (rate(messenger_job_queue_wait_seconds_bucket[5m])))", "p95 queue wait"),
("histogram_quantile(0.95, sum by (le) (rate(messenger_outbox_publish_delay_seconds_bucket[5m])))", "p95 outbox -> broker")],
"s", description="Backlog (сколько ждут) отдельно от processing latency (сколько обрабатываются).")
ts("Saturation worker: занятость потребителей", [
("sum(rate(messenger_worker_busy_time_seconds_total[1m])) / sum(last_over_time(messenger_worker_capacity[30s]))", "utilization (busy_time/capacity)"),
("sum(messenger_worker_busy) / sum(last_over_time(messenger_worker_capacity[30s]))", "busy/capacity (мгновенно)")],
"percentunit", description="1.0 = все WORKER_CONCURRENCY потоков всех реплик заняты; очередь растёт быстрее обработки.")
ts("Relay: публикации outbox", [
("sum by (outcome) (rate(messenger_outbox_published_total[1m]))", "{{outcome}}")], "short")
# ------------------------------------------------------------------ DB / S3
row("PostgreSQL и S3")
ts("Пул соединений API: used / idle / max", [
('sum by (instance, db_client_connection_state) (db_client_connection_count{%s})' % API,
"{{instance}} {{db_client_connection_state}}"),
('max(db_client_connection_max{%s})' % API, "max на реплику")], "short")
ts("Ожидающие соединение и таймауты пула", [
('sum by (instance) (db_client_connection_pending_requests{%s})' % API, "pending {{instance}}"),
('sum by (instance) (increase(db_client_connection_timeouts_total{%s}[1m]))' % API, "timeouts/мин {{instance}}")],
"short")
ts("Ожидание соединения пула p95", [
('histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket{%s}[1m])))' % API, "p95 wait")],
"s")
ts("Операции БД p95 по операции (API)", [
('histogram_quantile(0.95, sum by (le, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{%s}[1m])))' % API,
"{{db_operation_name}}")], "s", w=12)
ts("Операции БД worker/relay p95", [
('histogram_quantile(0.95, sum by (le, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{job=~"messenger-worker|messenger-relay"}[5m])))',
"{{db_operation_name}}")], "s", w=12)
ts("Ошибки операций БД", [
('sum by (job, db_operation_name, error_type) (rate(db_client_operation_duration_seconds_count{error_type!=""}[1m]))',
"{{job}} {{db_operation_name}} {{error_type}}")], "short")
ts("S3 p95 по операции", [
('histogram_quantile(0.95, sum by (le, messenger_s3_operation) (rate(messenger_s3_operation_duration_seconds_bucket[5m])))',
"{{messenger_s3_operation}}")], "s")
ts("S3 ошибки", [
('sum by (job, messenger_s3_operation, outcome) (rate(messenger_s3_operation_duration_seconds_count{outcome!="ok"}[1m]))',
"{{job}} {{messenger_s3_operation}} {{outcome}}")], "short")
# ------------------------------------------------------------------ ресурсы
row("Ресурсы контейнеров (CPU/RAM)")
ts("CPU, ядер (cgroup контейнера)", [
("sum by (job, instance) (rate(container_cpu_time_seconds_total[1m]))", "{{job}} {{instance}}")], "short", w=12)
ts("CPU saturation: доля лимита", [
("rate(container_cpu_time_seconds_total[1m]) / on(job, instance) container_cpu_limit", "{{job}} {{instance}}")],
"percentunit", w=12)
ts("Память контейнеров приложений", [
("container_memory_usage_bytes", "{{job}} {{instance}}")], "bytes", w=12)
ts("Память: доля лимита", [
("container_memory_usage_bytes / on(job, instance) container_memory_limit_bytes", "{{job}} {{instance}}")],
"percentunit", w=12)
ts("Память telemetry-стека (RSS)", [
('process_resident_memory_bytes{job=~"prometheus|loki|tempo"}', "{{job}}"),
('{__name__=~"otelcol_process_memory_rss(_bytes)?"}', "otel-collector RSS (вкл. страницы бинарника)"),
('otelcol_process_runtime_heap_alloc_bytes', "otel-collector Go heap"),
('rabbitmq_process_resident_memory_bytes', "rabbitmq")], "bytes", w=12)
ts("CPU telemetry-стека", [
('rate(process_cpu_seconds_total{job=~"prometheus|loki|tempo"}[1m])', "{{job}}"),
('rate({__name__=~"otelcol_process_cpu_seconds(_total)?"}[1m])', "otel-collector")], "short", w=12)
# ------------------------------------------------------------------ telemetry
row("Конвейер телеметрии (потери)")
ts("Collector: принято / отправлено / потеряно (spans)", [
('sum(rate({__name__=~"otelcol_receiver_accepted_spans(_total)?"}[1m]))', "принято"),
('sum(rate({__name__=~"otelcol_exporter_sent_spans(_total)?"}[1m]))', "отправлено в Tempo"),
('sum(rate({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[1m]))', "ошибка отправки"),
('sum(rate({__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?"}[1m]))', "очередь полна (потеря)"),
('sum(rate({__name__=~"otelcol_receiver_refused_spans(_total)?"}[1m]))', "отказ приёма (memory_limiter)")], "short")
ts("Collector: логи", [
('sum(rate({__name__=~"otelcol_receiver_accepted_log_records(_total)?"}[1m]))', "принято"),
('sum(rate({__name__=~"otelcol_exporter_sent_log_records(_total)?"}[1m]))', "отправлено в Loki"),
('sum(rate({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}[1m]))', "ошибка отправки"),
('sum(rate({__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?"}[1m]))', "очередь полна (потеря)")], "short")
ts("Collector: очередь экспортёров", [
("max by (exporter) (otelcol_exporter_queue_size)", "size {{exporter}}"),
("max by (exporter) (otelcol_exporter_queue_capacity)", "capacity {{exporter}}")], "short")
ts("SDK приложений: потери на стороне процесса", [
('sum by (job, error_type) (rate(otel_sdk_processor_span_processed_total{error_type!=""}[1m]))', "spans dropped {{job}} {{error_type}}"),
('sum by (job, error_type) (rate(otel_sdk_processor_log_processed_total{error_type!=""}[1m]))', "logs dropped {{job}} {{error_type}}"),
('sum by (job, error_type) (rate(otel_sdk_exporter_span_exported_total{error_type!=""}[1m]))', "span export failed {{job}} {{error_type}}")],
"short", w=12)
ts("SDK: заполнение очереди BatchSpanProcessor", [
("max by (job, instance) (otel_sdk_processor_span_queue_size)", "{{job}} {{instance}}"),
("max(otel_sdk_processor_span_queue_capacity)", "capacity")], "short", w=12)
# ------------------------------------------------------------------ logs / traces
row("Логи и трассы")
_panels.append({
"type": "logs", "id": _next_id(), "title": "Логи WARN/ERROR приложений (Loki)", "datasource": LOKI,
"gridPos": place(24, 10),
"targets": [{"datasource": LOKI, "refId": "A", "queryType": "range",
"expr": '{service_name=~"messenger-.+"} | severity_text=~"WARN.*|ERROR.*|CRITICAL"'}],
"options": {"showTime": True, "wrapLogMessage": True, "enableLogDetails": True, "sortOrder": "Descending",
"dedupStrategy": "none", "prettifyLogMessage": False}})
_panels.append({
"type": "logs", "id": _next_id(), "title": "HTTP 5xx (event=http.request)", "datasource": LOKI,
"gridPos": place(12, 10),
"targets": [{"datasource": LOKI, "refId": "A", "queryType": "range",
"expr": '{service_name="messenger-api"} | event="http.request" | http_response_status_code >= 500'}],
"options": {"showTime": True, "wrapLogMessage": True, "enableLogDetails": True, "sortOrder": "Descending"}})
_panels.append({
"type": "table", "id": _next_id(), "title": "Последние трассы обработки заданий (Tempo)", "datasource": TEMPO,
"gridPos": place(12, 10),
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces",
"query": '{ resource.service.name = "messenger-worker" && name = "attachments.process.v1 process" }'}]})
_panels.append({
"type": "table", "id": _next_id(), "title": "Медленные запросы API > 1 s (Tempo)", "datasource": TEMPO,
"gridPos": place(12, 10),
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces",
"query": '{ resource.service.name = "messenger-api" && kind = server && duration > 1s }'}]})
_panels.append({
"type": "table", "id": _next_id(), "title": "Трассы с ошибкой (Tempo)", "datasource": TEMPO,
"gridPos": place(12, 10),
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces",
"query": '{ status = error }'}]})
return {
"uid": "messenger-overview",
"title": "Messenger — наблюдаемость (lab6)",
"description": "API, worker, очередь, PostgreSQL, S3, ресурсы, конвейер телеметрии. Provisioning из Git.",
"tags": ["messenger", "lab6", "otel"],
"timezone": "browser",
"editable": False,
"graphTooltip": 1,
"refresh": "10s",
"schemaVersion": 39,
"version": 1,
"time": {"from": "now-30m", "to": "now"},
"templating": {"list": [{
"name": "instance", "label": "API-реплика", "type": "query", "datasource": PROM,
"query": {"query": 'label_values(http_server_request_duration_seconds_count{job="messenger-api"}, instance)',
"refId": "PrometheusVariableQueryEditor-VariableQuery"},
"definition": 'label_values(http_server_request_duration_seconds_count{job="messenger-api"}, instance)',
"includeAll": True, "multi": True, "allValue": ".*", "current": {"text": "All", "value": "$__all"},
"refresh": 2, "sort": 1}]},
"annotations": {"list": [{
"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, "enable": True, "hide": True,
"iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard"}]},
"panels": _panels,
}
if __name__ == "__main__":
OUT.parent.mkdir(parents=True, exist_ok=True)
OUT.write_text(json.dumps(build(), ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
print("written %s (%d panels)" % (OUT.relative_to(OUT.parents[3]), len(_panels)))