Files
mtusi2026lab7/scripts/gen_dashboard.py
T
alenchha 861e7722b5 ЛР 7 — БВТ2301 — Курлова — на 4
Перенесён минимум lab6 (API без изменений), добавлена защита на границах доверия:
- внешний HTTPS; обязательный mTLS LB -> API с allowlist identity и отзывом по serial;
- TLS на всех связях: PostgreSQL verify-full, MinIO HTTPS, RabbitMQ AMQPS mTLS (EXTERNAL),
  Valkey TLS + ACL, Collector и хранилища телеметрии через mTLS с отдельными CA;
- OpenBao в server mode: cert auth, токены 15 мин, политики только на свой путь;
- роли БД, S3-политики и права брокера по принципу минимальных прав;
- Argon2id с замером, rehash, общий лимит входа, бюджет защиты от login DoS;
- hardening всех контейнеров, scan образов с разбором, модель угроз, матрица доверия;
- отрицательные тесты: TLS-аудит (135 проверок), ACL, ротации, sealed OpenBao.
2026-11-16 10:49:13 +03:00

245 lines
15 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Generates infra/grafana/dashboards/messenger.json (provisioned into Grafana at startup).
The JSON is committed; this script only keeps it readable/editable. Run: python3 scripts/gen_dashboard.py
Every expression uses metric names that really exist in Prometheus (see REPORT.md, "Схема сигналов").
"""
import json
from pathlib import Path
OUT = Path(__file__).resolve().parents[1] / "infra/grafana/dashboards/messenger.json"
PROM = {"type": "prometheus", "uid": "prometheus"}
LOKI = {"type": "loki", "uid": "loki"}
TEMPO = {"type": "tempo", "uid": "tempo"}
API = 'service_name="messenger-api"'
HTTP = "http_server_request_duration_seconds"
_ids = iter(range(1, 1000))
def target(expr, legend="", ref="A", instant=False):
return {"datasource": PROM, "expr": expr, "legendFormat": legend, "refId": ref, "range": not instant,
"instant": instant}
def ts(title, targets, unit="short", x=0, y=0, w=12, h=8, desc="", stack=False, extra=None):
panel = {
"id": next(_ids), "type": "timeseries", "title": title, "description": desc, "datasource": PROM,
"gridPos": {"x": x, "y": y, "w": w, "h": h},
"fieldConfig": {"defaults": {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10,
"stacking": {"mode": "normal" if stack else "none"},
"showPoints": "never"}},
"overrides": []},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]},
"tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [target(*t) if isinstance(t, tuple) else t for t in targets],
}
if extra:
panel["fieldConfig"]["defaults"].update(extra)
return panel
def stat(title, expr, unit, x, y, w=4, h=4, desc="", thresholds=None, decimals=None):
steps = thresholds or [{"color": "green", "value": None}]
panel = {
"id": next(_ids), "type": "stat", "title": title, "description": desc, "datasource": PROM,
"gridPos": {"x": x, "y": y, "w": w, "h": h},
"fieldConfig": {"defaults": {"unit": unit, "thresholds": {"mode": "absolute", "steps": steps}},
"overrides": []},
"options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
"colorMode": "value", "graphMode": "area", "textMode": "auto"},
"targets": [target(expr, title)],
}
if decimals is not None:
panel["fieldConfig"]["defaults"]["decimals"] = decimals
return panel
def row(title, y):
return {"id": next(_ids), "type": "row", "title": title, "collapsed": False, "gridPos": {"x": 0, "y": y, "w": 24, "h": 1},
"panels": []}
def q(p, expr_inner):
return "histogram_quantile(%s, %s)" % (p, expr_inner)
panels = []
y = 0
# ------------------------------------------------------------------ SLO and alerts
panels.append(row("SLO (окно 30 мин) и алерты", y)); y += 1
red_yellow = lambda bad, warn: [{"color": "red", "value": None}, {"color": "yellow", "value": bad},
{"color": "green", "value": warn}]
panels += [
stat("Доступность (SLO 99%)", "messenger:slo_availability:ratio30m", "percentunit", 0, y,
desc="1 - 5xx/все бизнес-запросы messenger-api за 30 мин. Health-пробы не инструментированы.",
thresholds=red_yellow(0.99, 0.995), decimals=2),
stat("Латентность ≤500 ms (SLO 95%)", "messenger:slo_latency:ratio30m", "percentunit", 4, y,
desc="Доля запросов в bucket le=0.5 s за 30 мин.", thresholds=red_yellow(0.95, 0.97), decimals=2),
stat("Error ratio 5xx (2m)", "messenger:http_error_ratio:rate2m", "percentunit", 8, y,
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 0.05}], decimals=2),
stat("Старейшее задание", 'max(messenger_queue_oldest_age_seconds{queue="attachments.process"})', "s", 12, y,
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 60}]),
{"id": next(_ids), "type": "alertlist", "title": "Алерты Prometheus (firing/pending)",
"gridPos": {"x": 16, "y": y, "w": 8, "h": 8},
"options": {"alertInstanceLabelFilter": "", "alertName": "", "dashboardAlerts": False, "groupBy": [],
"groupMode": "default", "maxItems": 20, "sortOrder": 1, "stateFilter":
{"firing": True, "pending": True, "noData": False, "normal": False, "error": True},
"viewMode": "list", "datasource": "Prometheus"}},
]
y += 4
panels += [
stat("RPS API", "messenger:http_requests:rate2m", "reqps", 0, y, decimals=1),
stat("p95 API (5m)", "messenger:http_latency_p95:rate5m", "s", 4, y,
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}]),
stat("Загрузка worker", "messenger:worker_saturation:ratio", "percentunit", 8, y,
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 0.9}]),
stat("Ожидают пул БД", "max(db_client_connection_pending_requests)", "short", 12, y,
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}]),
]
y += 4
# ------------------------------------------------------------------ API
panels.append(row("API (messenger-api, все реплики)", y)); y += 1
rate_count = "rate(%s_count{%s}[$__rate_interval])" % (HTTP, API)
panels += [
ts("RPS по маршруту и методу", [("sum by (http_request_method, http_route) (%s)" % rate_count,
"{{http_request_method}} {{http_route}}")], "reqps", 0, y,
desc="Counter = _count гистограммы http.server.request.duration; route — шаблон, не реальный UUID."),
ts("RPS по классу статуса", [(
'sum by (status_class) (label_replace(%s, "status_class", "${1}xx", "http_response_status_code", "(\\\\d)..")) ' % rate_count,
"{{status_class}}")], "reqps", 12, y, stack=True),
]
y += 8
bucket = "sum by (le) (rate(%s_bucket{%s}[$__rate_interval]))" % (HTTP, API)
panels += [
ts("Латентность API p50/p95/p99 (из histogram buckets)", [
(q(0.5, bucket), "p50", "A"), (q(0.95, bucket), "p95", "B"), (q(0.99, bucket), "p99", "C")], "s", 0, y,
desc="histogram_quantile по сумме buckets всех реплик; не среднее и не усреднение p99 реплик."),
ts("Доля ошибок 5xx", [
('(sum(rate(%s_count{%s,http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(%s)' % (HTTP, API, rate_count),
"5xx / все", "A")], "percentunit", 12, y,
desc="Числитель: 5xx; знаменатель: все бизнес-запросы (health исключён). 4xx не считаются отказом."),
]
y += 8
panels += [
ts("p95 по маршруту", [(q(0.95, "sum by (le, http_route) (rate(%s_bucket{%s}[$__rate_interval]))" % (HTTP, API)),
"{{http_route}}")], "s", 0, y),
ts("Запросы в обработке", [("sum by (instance) (http_server_active_requests{%s})" % API, "{{instance}}")],
"short", 12, y),
]
y += 8
# ------------------------------------------------------------------ worker and queue
panels.append(row("Очередь и worker", y)); y += 1
jb = "sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"succeeded\"}[$__rate_interval]))"
rb = "sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))"
panels += [
ts("Jobs по исходу (в сек.)", [("sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))",
"{{outcome}}")], "ops", 0, y, stack=True,
desc="succeeded / retry / failed / noop / lost / dlq / requeued; job_type=attachment.process.v1"),
ts("Длительность job (succeeded) p50/p95/p99", [(q(0.5, jb), "p50", "A"), (q(0.95, jb), "p95", "B"),
(q(0.99, jb), "p99", "C")], "s", 12, y),
]
y += 8
panels += [
ts("Глубина очереди (из PostgreSQL)", [("max by (queue, state) (messenger_queue_depth)", "{{queue}} {{state}}")],
"short", 0, y, desc="max() по репликам API: каждая публикует одно и то же значение из БД."),
ts("Возраст старейшей job", [('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})', "oldest age")],
"s", 12, y, desc="От durable acceptance (COMMIT job) до сейчас; это backlog, не processing latency.",
extra={"custom": {"lineWidth": 2, "fillOpacity": 10, "showPoints": "never",
"thresholdsStyle": {"mode": "line"}},
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None},
{"color": "red", "value": 60}]}}),
]
y += 8
panels += [
ts("Время до ready p50/p95/p99", [(q(0.5, rb), "p50", "A"), (q(0.95, rb), "p95", "B"), (q(0.99, rb), "p99", "C")],
"s", 0, y, desc="messenger.attachment.time_to_ready: от 202 (COMMIT job) до COMMIT ready."),
ts("Насыщение worker: занято / ёмкость", [
("sum(messenger_worker_busy)", "busy", "A"), ("sum(messenger_worker_capacity)", "capacity", "B"),
("messenger:worker_saturation:ratio", "saturation", "C")], "short", 12, y),
]
y += 8
# ------------------------------------------------------------------ resources
panels.append(row("Ресурсы: пул БД, CPU, RAM", y)); y += 1
panels += [
ts("Пул БД: used / idle / waiting", [
("sum by (service_name, db_client_connection_state) (db_client_connection_count)",
"{{service_name}} {{db_client_connection_state}}", "A"),
("sum by (service_name) (db_client_connection_pending_requests)", "{{service_name}} waiting", "B"),
("sum by (service_name) (db_client_connection_max)", "{{service_name}} max", "C")], "short", 0, y),
ts("CPU процессов (ядра)", [
("sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))",
"{{service_name}} {{instance}}", "A"),
("rate(otelcol_process_cpu_seconds_total[$__rate_interval])", "otel-collector", "B")], "percentunit", 12, y,
desc="worker: self + завершённые дочерние процессы обработки изображений."),
]
y += 8
panels += [
ts("RSS процессов", [("process_memory_usage_bytes", "{{service_name}} {{instance}}", "A"),
("otelcol_process_memory_rss_bytes", "otel-collector", "B")], "bytes", 0, y),
ts("Telemetry pipeline: принято / отправлено / ошибки (items/s)", [
("sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval]))", "accepted spans", "A"),
("sum(rate(otelcol_receiver_accepted_log_records_total[$__rate_interval]))", "accepted log records", "E"),
("sum(rate(otelcol_receiver_accepted_metric_points_total[$__rate_interval]))", "accepted metric points", "F"),
("sum by (exporter) (rate({__name__=~\"otelcol_exporter_sent_(spans|log_records|metric_points)_total\"}[$__rate_interval]))",
"sent {{exporter}}", "B"),
# Separate queries: rate() drops __name__, so a regex over several failed_* counters collides once they all exist.
("sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval])) or vector(0)", "send failed spans", "C"),
("sum(rate(otelcol_exporter_enqueue_failed_spans_total[$__rate_interval])) or vector(0)", "dropped spans (queue full)", "G"),
("sum(rate(otelcol_exporter_send_failed_log_records_total[$__rate_interval])) or vector(0)", "send failed logs", "H"),
("sum(rate(otelcol_exporter_enqueue_failed_log_records_total[$__rate_interval])) or vector(0)", "dropped logs (queue full)", "I"),
("sum by (exporter) (otelcol_exporter_queue_size)", "queue {{exporter}}", "D")], "short", 12, y),
]
y += 8
# ------------------------------------------------------------------ logs and traces
panels.append(row("Логи и трассы", y)); y += 1
panels += [
{"id": next(_ids), "type": "logs", "title": "Логи API и worker (warning/error)", "datasource": LOKI,
"gridPos": {"x": 0, "y": y, "w": 12, "h": 10},
"options": {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True},
"targets": [{"datasource": LOKI, "refId": "A",
"expr": '{service_name=~"messenger-api|messenger-worker"} | severity_text=~"WARN.*|ERROR"'}]},
{"id": next(_ids), "type": "table", "title": "Трассы с ошибками или медленнее 500 ms (TraceQL)", "datasource": TEMPO,
"gridPos": {"x": 12, "y": y, "w": 12, "h": 10},
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20,
"query": '{resource.service.name=~"messenger-.*" && (status=error || duration>500ms)}'}]},
]
y += 10
dashboard = {
"uid": "messenger-lab6", "title": "Messenger — наблюдаемость (lab6)", "tags": ["messenger", "lab6"],
"timezone": "browser", "schemaVersion": 39, "version": 1, "editable": False, "refresh": "10s",
"time": {"from": "now-30m", "to": "now"}, "graphTooltip": 1, "panels": panels,
"annotations": {"list": [{"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"},
"enable": True, "hide": True, "name": "Annotations & Alerts", "type": "dashboard"}]},
"templating": {"list": []},
}
OUT.write_text(json.dumps(dashboard, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print("wrote %s (%d panels)" % (OUT.relative_to(OUT.parents[3]), len(panels)))
# Human-readable list of every panel: datasource, query and unit (kept in sync with the JSON by this script).
UNITS = {"reqps": "запросы/с", "percentunit": "доля (0..1)", "s": "секунды", "ops": "операции/с", "short": "штуки",
"bytes": "байты"}
DOC = OUT.parents[3] / "docs/dashboard.md"
lines = ["# Панели dashboard «%s»" % dashboard["title"], "",
"Сгенерировано `scripts/gen_dashboard.py` вместе с `infra/grafana/dashboards/messenger.json`. "
"`$__rate_interval` подставляет Grafana (не меньше 4 × scrape/export interval).", ""]
for panel in panels:
if panel["type"] == "row":
lines += ["## " + panel["title"], ""]
continue
unit = panel.get("fieldConfig", {}).get("defaults", {}).get("unit", "")
lines.append("**%s** (%s%s)" % (panel["title"], panel["type"], ", единица: " + UNITS.get(unit, unit) if unit else ""))
if panel.get("description"):
lines.append("— " + panel["description"])
lines.append("")
if panel["type"] == "alertlist":
lines += ["Правила Prometheus в состоянии pending/firing (`infra/prometheus/rules/messenger.yml`).", ""]
for t in panel.get("targets", []):
query = t.get("expr") or t.get("query")
legend = t.get("legendFormat", "")
lines += ["```%s" % ("promql" if t["datasource"]["type"] == "prometheus" else ""),
query + (" # " + legend if legend and legend != panel["title"] else ""), "```"]
lines.append("")
DOC.write_text("\n".join(lines), encoding="utf-8")
print("wrote docs/dashboard.md")