Files
mtusi2026lab6/scripts/gen_dashboard.py
alenchha 0638263029 ЛР 6 — БВТ2301 — Курлова — на 5 (трек 1 «Цена наблюдаемости»)
Перенесён минимум lab5 без изменения API и добавлена наблюдаемость:
- OTel SDK в API и worker: traces, metrics, logs по OTLP; W3C trace context
  API -> outbox -> RabbitMQ -> worker (producer/consumer spans, retry в той же трассе);
- OTel Collector (memory_limiter, scrub, batch, ограниченные очереди, профиль tail sampling),
  Prometheus, Loki, Tempo, Grafana с provisioning, dashboard и 6 alert rules с runbooks;
- проверки: наблюдаемость и приватность, инциденты A/B/C, насыщение worker,
  переполнение очереди exporter, эксперимент трека 1 (4 конфигурации x 3 прогона);
- CI job со сборкой и запуском стенда; REPORT.md с доказательствами в evidence/.
2026-11-11 09:24:27 +03:00

245 lines
15 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Generates infra/grafana/dashboards/messenger.json (provisioned into Grafana at startup).
The JSON is committed; this script only keeps it readable/editable. Run: python3 scripts/gen_dashboard.py
Every expression uses metric names that really exist in Prometheus (see REPORT.md, "Схема сигналов").
"""
import json
from pathlib import Path
OUT = Path(__file__).resolve().parents[1] / "infra/grafana/dashboards/messenger.json"
PROM = {"type": "prometheus", "uid": "prometheus"}
LOKI = {"type": "loki", "uid": "loki"}
TEMPO = {"type": "tempo", "uid": "tempo"}
API = 'service_name="messenger-api"'
HTTP = "http_server_request_duration_seconds"
_ids = iter(range(1, 1000))
def target(expr, legend="", ref="A", instant=False):
return {"datasource": PROM, "expr": expr, "legendFormat": legend, "refId": ref, "range": not instant,
"instant": instant}
def ts(title, targets, unit="short", x=0, y=0, w=12, h=8, desc="", stack=False, extra=None):
panel = {
"id": next(_ids), "type": "timeseries", "title": title, "description": desc, "datasource": PROM,
"gridPos": {"x": x, "y": y, "w": w, "h": h},
"fieldConfig": {"defaults": {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10,
"stacking": {"mode": "normal" if stack else "none"},
"showPoints": "never"}},
"overrides": []},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]},
"tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [target(*t) if isinstance(t, tuple) else t for t in targets],
}
if extra:
panel["fieldConfig"]["defaults"].update(extra)
return panel
def stat(title, expr, unit, x, y, w=4, h=4, desc="", thresholds=None, decimals=None):
steps = thresholds or [{"color": "green", "value": None}]
panel = {
"id": next(_ids), "type": "stat", "title": title, "description": desc, "datasource": PROM,
"gridPos": {"x": x, "y": y, "w": w, "h": h},
"fieldConfig": {"defaults": {"unit": unit, "thresholds": {"mode": "absolute", "steps": steps}},
"overrides": []},
"options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
"colorMode": "value", "graphMode": "area", "textMode": "auto"},
"targets": [target(expr, title)],
}
if decimals is not None:
panel["fieldConfig"]["defaults"]["decimals"] = decimals
return panel
def row(title, y):
return {"id": next(_ids), "type": "row", "title": title, "collapsed": False, "gridPos": {"x": 0, "y": y, "w": 24, "h": 1},
"panels": []}
def q(p, expr_inner):
return "histogram_quantile(%s, %s)" % (p, expr_inner)
panels = []
y = 0
# ------------------------------------------------------------------ SLO and alerts
panels.append(row("SLO (окно 30 мин) и алерты", y)); y += 1
red_yellow = lambda bad, warn: [{"color": "red", "value": None}, {"color": "yellow", "value": bad},
{"color": "green", "value": warn}]
panels += [
stat("Доступность (SLO 99%)", "messenger:slo_availability:ratio30m", "percentunit", 0, y,
desc="1 - 5xx/все бизнес-запросы messenger-api за 30 мин. Health-пробы не инструментированы.",
thresholds=red_yellow(0.99, 0.995), decimals=2),
stat("Латентность ≤500 ms (SLO 95%)", "messenger:slo_latency:ratio30m", "percentunit", 4, y,
desc="Доля запросов в bucket le=0.5 s за 30 мин.", thresholds=red_yellow(0.95, 0.97), decimals=2),
stat("Error ratio 5xx (2m)", "messenger:http_error_ratio:rate2m", "percentunit", 8, y,
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 0.05}], decimals=2),
stat("Старейшее задание", 'max(messenger_queue_oldest_age_seconds{queue="attachments.process"})', "s", 12, y,
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 60}]),
{"id": next(_ids), "type": "alertlist", "title": "Алерты Prometheus (firing/pending)",
"gridPos": {"x": 16, "y": y, "w": 8, "h": 8},
"options": {"alertInstanceLabelFilter": "", "alertName": "", "dashboardAlerts": False, "groupBy": [],
"groupMode": "default", "maxItems": 20, "sortOrder": 1, "stateFilter":
{"firing": True, "pending": True, "noData": False, "normal": False, "error": True},
"viewMode": "list", "datasource": "Prometheus"}},
]
y += 4
panels += [
stat("RPS API", "messenger:http_requests:rate2m", "reqps", 0, y, decimals=1),
stat("p95 API (5m)", "messenger:http_latency_p95:rate5m", "s", 4, y,
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}]),
stat("Загрузка worker", "messenger:worker_saturation:ratio", "percentunit", 8, y,
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 0.9}]),
stat("Ожидают пул БД", "max(db_client_connection_pending_requests)", "short", 12, y,
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}]),
]
y += 4
# ------------------------------------------------------------------ API
panels.append(row("API (messenger-api, все реплики)", y)); y += 1
rate_count = "rate(%s_count{%s}[$__rate_interval])" % (HTTP, API)
panels += [
ts("RPS по маршруту и методу", [("sum by (http_request_method, http_route) (%s)" % rate_count,
"{{http_request_method}} {{http_route}}")], "reqps", 0, y,
desc="Counter = _count гистограммы http.server.request.duration; route — шаблон, не реальный UUID."),
ts("RPS по классу статуса", [(
'sum by (status_class) (label_replace(%s, "status_class", "${1}xx", "http_response_status_code", "(\\\\d)..")) ' % rate_count,
"{{status_class}}")], "reqps", 12, y, stack=True),
]
y += 8
bucket = "sum by (le) (rate(%s_bucket{%s}[$__rate_interval]))" % (HTTP, API)
panels += [
ts("Латентность API p50/p95/p99 (из histogram buckets)", [
(q(0.5, bucket), "p50", "A"), (q(0.95, bucket), "p95", "B"), (q(0.99, bucket), "p99", "C")], "s", 0, y,
desc="histogram_quantile по сумме buckets всех реплик; не среднее и не усреднение p99 реплик."),
ts("Доля ошибок 5xx", [
('(sum(rate(%s_count{%s,http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(%s)' % (HTTP, API, rate_count),
"5xx / все", "A")], "percentunit", 12, y,
desc="Числитель: 5xx; знаменатель: все бизнес-запросы (health исключён). 4xx не считаются отказом."),
]
y += 8
panels += [
ts("p95 по маршруту", [(q(0.95, "sum by (le, http_route) (rate(%s_bucket{%s}[$__rate_interval]))" % (HTTP, API)),
"{{http_route}}")], "s", 0, y),
ts("Запросы в обработке", [("sum by (instance) (http_server_active_requests{%s})" % API, "{{instance}}")],
"short", 12, y),
]
y += 8
# ------------------------------------------------------------------ worker and queue
panels.append(row("Очередь и worker", y)); y += 1
jb = "sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"succeeded\"}[$__rate_interval]))"
rb = "sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))"
panels += [
ts("Jobs по исходу (в сек.)", [("sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))",
"{{outcome}}")], "ops", 0, y, stack=True,
desc="succeeded / retry / failed / noop / lost / dlq / requeued; job_type=attachment.process.v1"),
ts("Длительность job (succeeded) p50/p95/p99", [(q(0.5, jb), "p50", "A"), (q(0.95, jb), "p95", "B"),
(q(0.99, jb), "p99", "C")], "s", 12, y),
]
y += 8
panels += [
ts("Глубина очереди (из PostgreSQL)", [("max by (queue, state) (messenger_queue_depth)", "{{queue}} {{state}}")],
"short", 0, y, desc="max() по репликам API: каждая публикует одно и то же значение из БД."),
ts("Возраст старейшей job", [('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})', "oldest age")],
"s", 12, y, desc="От durable acceptance (COMMIT job) до сейчас; это backlog, не processing latency.",
extra={"custom": {"lineWidth": 2, "fillOpacity": 10, "showPoints": "never",
"thresholdsStyle": {"mode": "line"}},
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None},
{"color": "red", "value": 60}]}}),
]
y += 8
panels += [
ts("Время до ready p50/p95/p99", [(q(0.5, rb), "p50", "A"), (q(0.95, rb), "p95", "B"), (q(0.99, rb), "p99", "C")],
"s", 0, y, desc="messenger.attachment.time_to_ready: от 202 (COMMIT job) до COMMIT ready."),
ts("Насыщение worker: занято / ёмкость", [
("sum(messenger_worker_busy)", "busy", "A"), ("sum(messenger_worker_capacity)", "capacity", "B"),
("messenger:worker_saturation:ratio", "saturation", "C")], "short", 12, y),
]
y += 8
# ------------------------------------------------------------------ resources
panels.append(row("Ресурсы: пул БД, CPU, RAM", y)); y += 1
panels += [
ts("Пул БД: used / idle / waiting", [
("sum by (service_name, db_client_connection_state) (db_client_connection_count)",
"{{service_name}} {{db_client_connection_state}}", "A"),
("sum by (service_name) (db_client_connection_pending_requests)", "{{service_name}} waiting", "B"),
("sum by (service_name) (db_client_connection_max)", "{{service_name}} max", "C")], "short", 0, y),
ts("CPU процессов (ядра)", [
("sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))",
"{{service_name}} {{instance}}", "A"),
("rate(otelcol_process_cpu_seconds_total[$__rate_interval])", "otel-collector", "B")], "percentunit", 12, y,
desc="worker: self + завершённые дочерние процессы обработки изображений."),
]
y += 8
panels += [
ts("RSS процессов", [("process_memory_usage_bytes", "{{service_name}} {{instance}}", "A"),
("otelcol_process_memory_rss_bytes", "otel-collector", "B")], "bytes", 0, y),
ts("Telemetry pipeline: принято / отправлено / ошибки (items/s)", [
("sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval]))", "accepted spans", "A"),
("sum(rate(otelcol_receiver_accepted_log_records_total[$__rate_interval]))", "accepted log records", "E"),
("sum(rate(otelcol_receiver_accepted_metric_points_total[$__rate_interval]))", "accepted metric points", "F"),
("sum by (exporter) (rate({__name__=~\"otelcol_exporter_sent_(spans|log_records|metric_points)_total\"}[$__rate_interval]))",
"sent {{exporter}}", "B"),
# Separate queries: rate() drops __name__, so a regex over several failed_* counters collides once they all exist.
("sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval])) or vector(0)", "send failed spans", "C"),
("sum(rate(otelcol_exporter_enqueue_failed_spans_total[$__rate_interval])) or vector(0)", "dropped spans (queue full)", "G"),
("sum(rate(otelcol_exporter_send_failed_log_records_total[$__rate_interval])) or vector(0)", "send failed logs", "H"),
("sum(rate(otelcol_exporter_enqueue_failed_log_records_total[$__rate_interval])) or vector(0)", "dropped logs (queue full)", "I"),
("sum by (exporter) (otelcol_exporter_queue_size)", "queue {{exporter}}", "D")], "short", 12, y),
]
y += 8
# ------------------------------------------------------------------ logs and traces
panels.append(row("Логи и трассы", y)); y += 1
panels += [
{"id": next(_ids), "type": "logs", "title": "Логи API и worker (warning/error)", "datasource": LOKI,
"gridPos": {"x": 0, "y": y, "w": 12, "h": 10},
"options": {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True},
"targets": [{"datasource": LOKI, "refId": "A",
"expr": '{service_name=~"messenger-api|messenger-worker"} | severity_text=~"WARN.*|ERROR"'}]},
{"id": next(_ids), "type": "table", "title": "Трассы с ошибками или медленнее 500 ms (TraceQL)", "datasource": TEMPO,
"gridPos": {"x": 12, "y": y, "w": 12, "h": 10},
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20,
"query": '{resource.service.name=~"messenger-.*" && (status=error || duration>500ms)}'}]},
]
y += 10
dashboard = {
"uid": "messenger-lab6", "title": "Messenger — наблюдаемость (lab6)", "tags": ["messenger", "lab6"],
"timezone": "browser", "schemaVersion": 39, "version": 1, "editable": False, "refresh": "10s",
"time": {"from": "now-30m", "to": "now"}, "graphTooltip": 1, "panels": panels,
"annotations": {"list": [{"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"},
"enable": True, "hide": True, "name": "Annotations & Alerts", "type": "dashboard"}]},
"templating": {"list": []},
}
OUT.write_text(json.dumps(dashboard, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print("wrote %s (%d panels)" % (OUT.relative_to(OUT.parents[3]), len(panels)))
# Human-readable list of every panel: datasource, query and unit (kept in sync with the JSON by this script).
UNITS = {"reqps": "запросы/с", "percentunit": "доля (0..1)", "s": "секунды", "ops": "операции/с", "short": "штуки",
"bytes": "байты"}
DOC = OUT.parents[3] / "docs/dashboard.md"
lines = ["# Панели dashboard «%s»" % dashboard["title"], "",
"Сгенерировано `scripts/gen_dashboard.py` вместе с `infra/grafana/dashboards/messenger.json`. "
"`$__rate_interval` подставляет Grafana (не меньше 4 × scrape/export interval).", ""]
for panel in panels:
if panel["type"] == "row":
lines += ["## " + panel["title"], ""]
continue
unit = panel.get("fieldConfig", {}).get("defaults", {}).get("unit", "")
lines.append("**%s** (%s%s)" % (panel["title"], panel["type"], ", единица: " + UNITS.get(unit, unit) if unit else ""))
if panel.get("description"):
lines.append("— " + panel["description"])
lines.append("")
if panel["type"] == "alertlist":
lines += ["Правила Prometheus в состоянии pending/firing (`infra/prometheus/rules/messenger.yml`).", ""]
for t in panel.get("targets", []):
query = t.get("expr") or t.get("query")
legend = t.get("legendFormat", "")
lines += ["```%s" % ("promql" if t["datasource"]["type"] == "prometheus" else ""),
query + (" # " + legend if legend and legend != panel["title"] else ""), "```"]
lines.append("")
DOC.write_text("\n".join(lines), encoding="utf-8")
print("wrote docs/dashboard.md")