Files
mtusi2026lab7/scripts/build_dashboard.py
T
zemiles 9cc80d2529
Application / app (pull_request) Canceled after 0s
Skeleton contracts / contracts (pull_request) Canceled after 0s
ЛР 7 — БПИ2303 — Лазарев — на 3
Перенесена реализация lab6; бизнес-API без изменений, транспорт — HTTPS.

- Внешний HTTPS (public-ca) на 127.0.0.1:8443; HSTS, TLS >= 1.2.
- Обязательное mTLS LB -> каждая API-реплика: API проверяет цепочку
  internal-ca и SAN клиента (allowlist: lb); health только на 127.0.0.1.
- TLS с проверкой сервера ко всем зависимостям: PostgreSQL verify-full
  (pg_hba только hostssl), HTTPS S3, AMQPS, OTLP/HTTPS с client cert,
  mTLS Collector -> Prometheus/Loki/Tempo, Grafana -> datasources, scrape.
- OpenBao 2.7.2 не в dev-mode: Raft на volume, init/unseal оператором,
  auth/cert с ролью и policy на сервис, TTL 15 мин + renew/re-login.
- Отдельные роли PostgreSQL с грантами по таблицам; разные ключи S3 и
  пользователи брокера выдаются из OpenBao.
- Argon2id (m=19 MiB, t=2, p=1, замер на стенде) и rehash при входе.
- Случайный X-Instance-Id; hardening контейнеров (non-root, cap_drop ALL,
  no-new-privileges, read-only, лимиты); TCP_NODELAY на сокете API.
- Модель угроз, trust matrix, отрицательные тесты TLS/mTLS, API, OpenBao,
  hardening; evidence, в т.ч. прогон из чистого clone.
2026-11-18 23:59:35 +03:00

324 lines
23 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Генератор импортируемого dashboard Grafana: infra/grafana/dashboards/messenger.json.
JSON лежит в Git и подхватывается provisioning-ом; этот скрипт — воспроизводимый способ его
изменить (python3 scripts/build_dashboard.py). Только стандартная библиотека.
Все запросы ссылаются на реальные имена метрик (см. REPORT.md «Схема сигналов»).
Перцентили — histogram_quantile по sum by (le) buckets всех реплик, не среднее p95.
"""
import json
from pathlib import Path
OUT = Path(__file__).resolve().parents[1] / "infra/grafana/dashboards/messenger.json"
PROM = {"type": "prometheus", "uid": "prometheus"}
LOKI = {"type": "loki", "uid": "loki"}
TEMPO = {"type": "tempo", "uid": "tempo"}
API = 'job="messenger-api", instance=~"$instance"'
BUSINESS = API + ', http_route!~"/health/.*|unmatched"'
_panels = []
_id = [0]
_pos = {"x": 0, "y": 0, "h": 0}
def _next_id():
_id[0] += 1
return _id[0]
def place(w, h):
if _pos["x"] + w > 24:
_pos["x"] = 0
_pos["y"] += _pos["h"]
_pos["h"] = 0
g = {"x": _pos["x"], "y": _pos["y"], "w": w, "h": h}
_pos["x"] += w
_pos["h"] = max(_pos["h"], h)
return g
def row(title):
_pos["x"] = 0
_pos["y"] += _pos["h"]
_pos["h"] = 0
_panels.append({"type": "row", "id": _next_id(), "title": title, "collapsed": False,
"gridPos": {"x": 0, "y": _pos["y"], "w": 24, "h": 1}, "panels": []})
_pos["y"] += 1
def target(expr, legend="", ref="A", exemplar=False, instant=False):
t = {"datasource": PROM, "expr": expr, "legendFormat": legend or "__auto", "refId": ref,
"range": not instant, "instant": instant}
if exemplar:
t["exemplar"] = True
return t
def ts(title, targets, unit="short", w=8, h=8, description="", stack=False, min0=True, thresholds=None):
refs = "ABCDEFGHIJ"
tg = [target(*t[:2], ref=refs[i], exemplar=(len(t) > 2 and t[2])) for i, t in enumerate(targets)]
p = {"type": "timeseries", "id": _next_id(), "title": title, "description": description,
"datasource": PROM, "gridPos": place(w, h), "targets": tg,
"fieldConfig": {"defaults": {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10,
"showPoints": "never",
"stacking": {"mode": "normal" if stack else "none"}},
**({"min": 0} if min0 else {})},
"overrides": []},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]},
"tooltip": {"mode": "multi", "sort": "desc"}}}
if thresholds:
p["fieldConfig"]["defaults"]["thresholds"] = {"mode": "absolute", "steps": thresholds}
p["fieldConfig"]["defaults"]["custom"]["thresholdsStyle"] = {"mode": "line+area"}
_panels.append(p)
def stat(title, expr, unit="short", w=3, h=4, steps=None, description="", decimals=None):
p = {"type": "stat", "id": _next_id(), "title": title, "description": description, "datasource": PROM,
"gridPos": place(w, h), "targets": [target(expr, "", instant=True)],
"fieldConfig": {"defaults": {"unit": unit, "thresholds": {"mode": "absolute", "steps": steps or [
{"color": "green", "value": None}]}}, "overrides": []},
"options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
"colorMode": "background", "graphMode": "area", "textMode": "value"}}
if decimals is not None:
p["fieldConfig"]["defaults"]["decimals"] = decimals
_panels.append(p)
def build():
# ------------------------------------------------------------------ обзор
row("Обзор и SLO")
stat("RPS бизнес-запросов", "messenger:http_requests:rate2m", "reqps",
description="sum(rate(http_server_request_duration_seconds_count{job=messenger-api, без /health}[2m]))")
stat("5xx, доля (2m)", "messenger:http_error_ratio:rate2m", "percentunit", decimals=2,
steps=[{"color": "green", "value": None}, {"color": "orange", "value": 0.01}, {"color": "red", "value": 0.05}],
description="Числитель: 5xx бизнес-запросов; знаменатель: все бизнес-запросы (без /health, unmatched).")
stat("p95 API (5m)", "messenger:http_latency_seconds:p95_5m", "s", decimals=3,
steps=[{"color": "green", "value": None}, {"color": "orange", "value": 0.5}, {"color": "red", "value": 1}])
stat("SLO доступности, 7d", "messenger:slo_availability:ratio_7d", "percentunit", decimals=3,
steps=[{"color": "red", "value": None}, {"color": "green", "value": 0.995}],
description="SLO: 99.5 % бизнес-запросов без 5xx за скользящие 7 дней.")
stat("Остаток error budget, 7d", "messenger:slo_availability:error_budget_remaining_7d", "percentunit",
decimals=1, steps=[{"color": "red", "value": None}, {"color": "orange", "value": 0.25},
{"color": "green", "value": 0.5}])
stat("Time to ready p95", "messenger:time_to_ready_seconds:p95_5m", "s", decimals=2,
steps=[{"color": "green", "value": None}, {"color": "orange", "value": 10}, {"color": "red", "value": 30}],
description="От durable acceptance (COMMIT enqueue) до COMMIT ready. SLO: 99 % не дольше 30 с.")
stat("Ready lag очереди", "max(messenger_jobs_ready_lag_seconds) or vector(0)", "s", decimals=0,
steps=[{"color": "green", "value": None}, {"color": "orange", "value": 10}, {"color": "red", "value": 30}],
description="Сколько ждёт worker самое старое созревшее задание (PostgreSQL jobs).")
stat("Alerts firing", 'count(ALERTS{alertstate="firing"}) or vector(0)', "short",
steps=[{"color": "green", "value": None}, {"color": "red", "value": 1}])
_panels.append({
"type": "state-timeline", "id": _next_id(), "title": "История alerts (pending/firing -> resolved)",
"datasource": PROM, "gridPos": place(24, 6),
"targets": [target('max by (alertname, source, alertstate) (ALERTS)', "{{alertname}} {{source}} {{alertstate}}")],
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"},
"thresholds": {"mode": "absolute", "steps": [
{"color": "transparent", "value": None}, {"color": "red", "value": 1}]}},
"overrides": []},
"options": {"showValue": "never", "mergeValues": True, "rowHeight": 0.8,
"legend": {"showLegend": False}}})
# ------------------------------------------------------------------ API
row("API: трафик, ошибки, задержка")
ts("RPS по маршруту", [("sum by (http_route) (rate(http_server_request_duration_seconds_count{%s}[1m]))" % API,
"{{http_route}}")], "reqps", stack=True)
ts("Ответы по классу статуса", [(
"sum by (http_response_status_class) (rate(http_server_request_duration_seconds_count{%s}[1m]))" % BUSINESS,
"{{http_response_status_class}}")], "reqps", stack=True)
ts("Доля 5xx: API и LB", [
("messenger:http_error_ratio:rate2m", "API (без /health)"),
("messenger:lb_error_ratio:rate2m", "HAProxy fe_api")], "percentunit",
thresholds=[{"color": "transparent", "value": None}, {"color": "red", "value": 0.05}],
description="Если БД недоступна, реплики не ready и 503 отдаёт LB: эти запросы видны только в метрике LB.")
ts("Задержка API p50/p95/p99 (из histogram)", [
("histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p50", True),
("histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p95", True),
("histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p99", True)],
"s", w=12, description="Точки-exemplars ведут в трассу Tempo конкретного запроса.")
ts("p95 по маршруту", [(
"histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{%s}[5m])))" % BUSINESS,
"{{http_route}}")], "s", w=12)
ts("5xx по причине (error.type)", [(
'sum by (error_type, http_route) (rate(http_server_request_duration_seconds_count{%s, http_response_status_class="5xx"}[1m]))' % API,
"{{error_type}} {{http_route}}")], "reqps", stack=True)
ts("Запросы в обработке (saturation API)", [
("sum by (instance) (http_server_active_requests{%s})" % API, "{{instance}}")], "short")
ts("HAProxy: UP-реплики и очередь LB", [
('haproxy_backend_active_servers{proxy="be_api"}', "UP реплик"),
('haproxy_backend_current_queue{proxy="be_api"}', "очередь LB")], "short")
# ------------------------------------------------------------------ worker
row("Worker и очередь заданий")
ts("Задания по исходу (в минуту)", [(
"sum by (messenger_job_outcome) (increase(messenger_jobs_processed_total[1m]))", "{{messenger_job_outcome}}")],
"short", stack=True, description="ready / retry / failed / duplicate / lease_lost / rejected / requeued")
ts("Длительность обработки job p50/p95/p99", [
('histogram_quantile(0.50, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p50", True),
('histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p95", True),
('histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p99", True)],
"s")
ts("Time to ready p50/p95/p99", [
('histogram_quantile(0.50, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p50", True),
('histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p95", True),
('histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p99", True)],
"s", description="Backlog-ожидание + обработка: время от 202 (COMMIT) до ready.")
ts("Глубина очереди: PostgreSQL и RabbitMQ", [
('max by (messenger_job_status) (messenger_jobs_count{messenger_job_status=~"queued|processing"})', "jobs {{messenger_job_status}} (БД)"),
('max(messenger_outbox_pending)', "outbox не опубликовано"),
('sum(rabbitmq_detailed_queue_messages_ready{queue="attachments.process.v1"})', "RabbitMQ ready"),
('sum(rabbitmq_detailed_queue_messages_unacked{queue="attachments.process.v1"})', "RabbitMQ unacked"),
('sum(rabbitmq_detailed_queue_messages{queue="attachments.dlq"})', "DLQ")], "short")
ts("Возраст старейшего задания и lag", [
('max by (messenger_job_status) (messenger_jobs_oldest_age_seconds)', "oldest {{messenger_job_status}}"),
("max(messenger_jobs_ready_lag_seconds)", "ready lag"),
("max(messenger_outbox_oldest_age_seconds)", "outbox oldest")], "s",
thresholds=[{"color": "transparent", "value": None}, {"color": "red", "value": 30}])
ts("Ожидание в очереди p95 (до claim)", [
("histogram_quantile(0.95, sum by (le) (rate(messenger_job_queue_wait_seconds_bucket[5m])))", "p95 queue wait"),
("histogram_quantile(0.95, sum by (le) (rate(messenger_outbox_publish_delay_seconds_bucket[5m])))", "p95 outbox -> broker")],
"s", description="Backlog (сколько ждут) отдельно от processing latency (сколько обрабатываются).")
ts("Saturation worker: занятость потребителей", [
("sum(rate(messenger_worker_busy_time_seconds_total[1m])) / sum(last_over_time(messenger_worker_capacity[30s]))", "utilization (busy_time/capacity)"),
("sum(messenger_worker_busy) / sum(last_over_time(messenger_worker_capacity[30s]))", "busy/capacity (мгновенно)")],
"percentunit", description="1.0 = все WORKER_CONCURRENCY потоков всех реплик заняты; очередь растёт быстрее обработки.")
ts("Relay: публикации outbox", [
("sum by (outcome) (rate(messenger_outbox_published_total[1m]))", "{{outcome}}")], "short")
# ------------------------------------------------------------------ DB / S3
row("PostgreSQL и S3")
ts("Пул соединений API: used / idle / max", [
('sum by (instance, db_client_connection_state) (db_client_connection_count{%s})' % API,
"{{instance}} {{db_client_connection_state}}"),
('max(db_client_connection_max{%s})' % API, "max на реплику")], "short")
ts("Ожидающие соединение и таймауты пула", [
('sum by (instance) (db_client_connection_pending_requests{%s})' % API, "pending {{instance}}"),
('sum by (instance) (increase(db_client_connection_timeouts_total{%s}[1m]))' % API, "timeouts/мин {{instance}}")],
"short")
ts("Ожидание соединения пула p95", [
('histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket{%s}[1m])))' % API, "p95 wait")],
"s")
ts("Операции БД p95 по операции (API)", [
('histogram_quantile(0.95, sum by (le, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{%s}[1m])))' % API,
"{{db_operation_name}}")], "s", w=12)
ts("Операции БД worker/relay p95", [
('histogram_quantile(0.95, sum by (le, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{job=~"messenger-worker|messenger-relay"}[5m])))',
"{{db_operation_name}}")], "s", w=12)
ts("Ошибки операций БД", [
('sum by (job, db_operation_name, error_type) (rate(db_client_operation_duration_seconds_count{error_type!=""}[1m]))',
"{{job}} {{db_operation_name}} {{error_type}}")], "short")
ts("S3 p95 по операции", [
('histogram_quantile(0.95, sum by (le, messenger_s3_operation) (rate(messenger_s3_operation_duration_seconds_bucket[5m])))',
"{{messenger_s3_operation}}")], "s")
ts("S3 ошибки", [
('sum by (job, messenger_s3_operation, outcome) (rate(messenger_s3_operation_duration_seconds_count{outcome!="ok"}[1m]))',
"{{job}} {{messenger_s3_operation}} {{outcome}}")], "short")
# ------------------------------------------------------------------ ресурсы
row("Ресурсы контейнеров (CPU/RAM)")
ts("CPU, ядер (cgroup контейнера)", [
("sum by (job, instance) (rate(container_cpu_time_seconds_total[1m]))", "{{job}} {{instance}}")], "short", w=12)
ts("CPU saturation: доля лимита", [
("rate(container_cpu_time_seconds_total[1m]) / on(job, instance) container_cpu_limit", "{{job}} {{instance}}")],
"percentunit", w=12)
ts("Память контейнеров приложений", [
("container_memory_usage_bytes", "{{job}} {{instance}}")], "bytes", w=12)
ts("Память: доля лимита", [
("container_memory_usage_bytes / on(job, instance) container_memory_limit_bytes", "{{job}} {{instance}}")],
"percentunit", w=12)
ts("Память telemetry-стека (RSS)", [
('process_resident_memory_bytes{job=~"prometheus|loki|tempo"}', "{{job}}"),
('{__name__=~"otelcol_process_memory_rss(_bytes)?"}', "otel-collector RSS (вкл. страницы бинарника)"),
('otelcol_process_runtime_heap_alloc_bytes', "otel-collector Go heap"),
('rabbitmq_process_resident_memory_bytes', "rabbitmq")], "bytes", w=12)
ts("CPU telemetry-стека", [
('rate(process_cpu_seconds_total{job=~"prometheus|loki|tempo"}[1m])', "{{job}}"),
('rate({__name__=~"otelcol_process_cpu_seconds(_total)?"}[1m])', "otel-collector")], "short", w=12)
# ------------------------------------------------------------------ telemetry
row("Конвейер телеметрии (потери)")
ts("Collector: принято / отправлено / потеряно (spans)", [
('sum(rate({__name__=~"otelcol_receiver_accepted_spans(_total)?"}[1m]))', "принято"),
('sum(rate({__name__=~"otelcol_exporter_sent_spans(_total)?"}[1m]))', "отправлено в Tempo"),
('sum(rate({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[1m]))', "ошибка отправки"),
('sum(rate({__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?"}[1m]))', "очередь полна (потеря)"),
('sum(rate({__name__=~"otelcol_receiver_refused_spans(_total)?"}[1m]))', "отказ приёма (memory_limiter)")], "short")
ts("Collector: логи", [
('sum(rate({__name__=~"otelcol_receiver_accepted_log_records(_total)?"}[1m]))', "принято"),
('sum(rate({__name__=~"otelcol_exporter_sent_log_records(_total)?"}[1m]))', "отправлено в Loki"),
('sum(rate({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}[1m]))', "ошибка отправки"),
('sum(rate({__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?"}[1m]))', "очередь полна (потеря)")], "short")
ts("Collector: очередь экспортёров", [
("max by (exporter) (otelcol_exporter_queue_size)", "size {{exporter}}"),
("max by (exporter) (otelcol_exporter_queue_capacity)", "capacity {{exporter}}")], "short")
ts("SDK приложений: потери на стороне процесса", [
('sum by (job, error_type) (rate(otel_sdk_processor_span_processed_total{error_type!=""}[1m]))', "spans dropped {{job}} {{error_type}}"),
('sum by (job, error_type) (rate(otel_sdk_processor_log_processed_total{error_type!=""}[1m]))', "logs dropped {{job}} {{error_type}}"),
('sum by (job, error_type) (rate(otel_sdk_exporter_span_exported_total{error_type!=""}[1m]))', "span export failed {{job}} {{error_type}}")],
"short", w=12)
ts("SDK: заполнение очереди BatchSpanProcessor", [
("max by (job, instance) (otel_sdk_processor_span_queue_size)", "{{job}} {{instance}}"),
("max(otel_sdk_processor_span_queue_capacity)", "capacity")], "short", w=12)
# ------------------------------------------------------------------ logs / traces
row("Логи и трассы")
_panels.append({
"type": "logs", "id": _next_id(), "title": "Логи WARN/ERROR приложений (Loki)", "datasource": LOKI,
"gridPos": place(24, 10),
"targets": [{"datasource": LOKI, "refId": "A", "queryType": "range",
"expr": '{service_name=~"messenger-.+"} | severity_text=~"WARN.*|ERROR.*|CRITICAL"'}],
"options": {"showTime": True, "wrapLogMessage": True, "enableLogDetails": True, "sortOrder": "Descending",
"dedupStrategy": "none", "prettifyLogMessage": False}})
_panels.append({
"type": "logs", "id": _next_id(), "title": "HTTP 5xx (event=http.request)", "datasource": LOKI,
"gridPos": place(12, 10),
"targets": [{"datasource": LOKI, "refId": "A", "queryType": "range",
"expr": '{service_name="messenger-api"} | event="http.request" | http_response_status_code >= 500'}],
"options": {"showTime": True, "wrapLogMessage": True, "enableLogDetails": True, "sortOrder": "Descending"}})
_panels.append({
"type": "table", "id": _next_id(), "title": "Последние трассы обработки заданий (Tempo)", "datasource": TEMPO,
"gridPos": place(12, 10),
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces",
"query": '{ resource.service.name = "messenger-worker" && name = "attachments.process.v1 process" }'}]})
_panels.append({
"type": "table", "id": _next_id(), "title": "Медленные запросы API > 1 s (Tempo)", "datasource": TEMPO,
"gridPos": place(12, 10),
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces",
"query": '{ resource.service.name = "messenger-api" && kind = server && duration > 1s }'}]})
_panels.append({
"type": "table", "id": _next_id(), "title": "Трассы с ошибкой (Tempo)", "datasource": TEMPO,
"gridPos": place(12, 10),
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces",
"query": '{ status = error }'}]})
return {
"uid": "messenger-overview",
"title": "Messenger — наблюдаемость (lab6)",
"description": "API, worker, очередь, PostgreSQL, S3, ресурсы, конвейер телеметрии. Provisioning из Git.",
"tags": ["messenger", "lab6", "otel"],
"timezone": "browser",
"editable": False,
"graphTooltip": 1,
"refresh": "10s",
"schemaVersion": 39,
"version": 1,
"time": {"from": "now-30m", "to": "now"},
"templating": {"list": [{
"name": "instance", "label": "API-реплика", "type": "query", "datasource": PROM,
"query": {"query": 'label_values(http_server_request_duration_seconds_count{job="messenger-api"}, instance)',
"refId": "PrometheusVariableQueryEditor-VariableQuery"},
"definition": 'label_values(http_server_request_duration_seconds_count{job="messenger-api"}, instance)',
"includeAll": True, "multi": True, "allValue": ".*", "current": {"text": "All", "value": "$__all"},
"refresh": 2, "sort": 1}]},
"annotations": {"list": [{
"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, "enable": True, "hide": True,
"iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard"}]},
"panels": _panels,
}
if __name__ == "__main__":
OUT.parent.mkdir(parents=True, exist_ok=True)
OUT.write_text(json.dumps(build(), ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
print("written %s (%d panels)" % (OUT.relative_to(OUT.parents[3]), len(_panels)))