forked from mtusi2026/mtusi2026lab6
Перенесён минимум lab5 без изменения API и добавлена наблюдаемость: - OTel SDK в API и worker: traces, metrics, logs по OTLP; W3C trace context API -> outbox -> RabbitMQ -> worker (producer/consumer spans, retry в той же трассе); - OTel Collector (memory_limiter, scrub, batch, ограниченные очереди, профиль tail sampling), Prometheus, Loki, Tempo, Grafana с provisioning, dashboard и 6 alert rules с runbooks; - проверки: наблюдаемость и приватность, инциденты A/B/C, насыщение worker, переполнение очереди exporter, эксперимент трека 1 (4 конфигурации x 3 прогона); - CI job со сборкой и запуском стенда; REPORT.md с доказательствами в evidence/.
245 lines
15 KiB
Python
245 lines
15 KiB
Python
#!/usr/bin/env python3
|
||
"""Generates infra/grafana/dashboards/messenger.json (provisioned into Grafana at startup).
|
||
|
||
The JSON is committed; this script only keeps it readable/editable. Run: python3 scripts/gen_dashboard.py
|
||
Every expression uses metric names that really exist in Prometheus (see REPORT.md, "Схема сигналов").
|
||
"""
|
||
import json
|
||
from pathlib import Path
|
||
|
||
OUT = Path(__file__).resolve().parents[1] / "infra/grafana/dashboards/messenger.json"
|
||
PROM = {"type": "prometheus", "uid": "prometheus"}
|
||
LOKI = {"type": "loki", "uid": "loki"}
|
||
TEMPO = {"type": "tempo", "uid": "tempo"}
|
||
API = 'service_name="messenger-api"'
|
||
HTTP = "http_server_request_duration_seconds"
|
||
_ids = iter(range(1, 1000))
|
||
|
||
|
||
def target(expr, legend="", ref="A", instant=False):
|
||
return {"datasource": PROM, "expr": expr, "legendFormat": legend, "refId": ref, "range": not instant,
|
||
"instant": instant}
|
||
|
||
|
||
def ts(title, targets, unit="short", x=0, y=0, w=12, h=8, desc="", stack=False, extra=None):
|
||
panel = {
|
||
"id": next(_ids), "type": "timeseries", "title": title, "description": desc, "datasource": PROM,
|
||
"gridPos": {"x": x, "y": y, "w": w, "h": h},
|
||
"fieldConfig": {"defaults": {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10,
|
||
"stacking": {"mode": "normal" if stack else "none"},
|
||
"showPoints": "never"}},
|
||
"overrides": []},
|
||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]},
|
||
"tooltip": {"mode": "multi", "sort": "desc"}},
|
||
"targets": [target(*t) if isinstance(t, tuple) else t for t in targets],
|
||
}
|
||
if extra:
|
||
panel["fieldConfig"]["defaults"].update(extra)
|
||
return panel
|
||
|
||
|
||
def stat(title, expr, unit, x, y, w=4, h=4, desc="", thresholds=None, decimals=None):
|
||
steps = thresholds or [{"color": "green", "value": None}]
|
||
panel = {
|
||
"id": next(_ids), "type": "stat", "title": title, "description": desc, "datasource": PROM,
|
||
"gridPos": {"x": x, "y": y, "w": w, "h": h},
|
||
"fieldConfig": {"defaults": {"unit": unit, "thresholds": {"mode": "absolute", "steps": steps}},
|
||
"overrides": []},
|
||
"options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
|
||
"colorMode": "value", "graphMode": "area", "textMode": "auto"},
|
||
"targets": [target(expr, title)],
|
||
}
|
||
if decimals is not None:
|
||
panel["fieldConfig"]["defaults"]["decimals"] = decimals
|
||
return panel
|
||
|
||
|
||
def row(title, y):
|
||
return {"id": next(_ids), "type": "row", "title": title, "collapsed": False, "gridPos": {"x": 0, "y": y, "w": 24, "h": 1},
|
||
"panels": []}
|
||
|
||
|
||
def q(p, expr_inner):
|
||
return "histogram_quantile(%s, %s)" % (p, expr_inner)
|
||
|
||
|
||
panels = []
|
||
y = 0
|
||
# ------------------------------------------------------------------ SLO and alerts
|
||
panels.append(row("SLO (окно 30 мин) и алерты", y)); y += 1
|
||
red_yellow = lambda bad, warn: [{"color": "red", "value": None}, {"color": "yellow", "value": bad},
|
||
{"color": "green", "value": warn}]
|
||
panels += [
|
||
stat("Доступность (SLO 99%)", "messenger:slo_availability:ratio30m", "percentunit", 0, y,
|
||
desc="1 - 5xx/все бизнес-запросы messenger-api за 30 мин. Health-пробы не инструментированы.",
|
||
thresholds=red_yellow(0.99, 0.995), decimals=2),
|
||
stat("Латентность ≤500 ms (SLO 95%)", "messenger:slo_latency:ratio30m", "percentunit", 4, y,
|
||
desc="Доля запросов в bucket le=0.5 s за 30 мин.", thresholds=red_yellow(0.95, 0.97), decimals=2),
|
||
stat("Error ratio 5xx (2m)", "messenger:http_error_ratio:rate2m", "percentunit", 8, y,
|
||
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 0.05}], decimals=2),
|
||
stat("Старейшее задание", 'max(messenger_queue_oldest_age_seconds{queue="attachments.process"})', "s", 12, y,
|
||
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 60}]),
|
||
{"id": next(_ids), "type": "alertlist", "title": "Алерты Prometheus (firing/pending)",
|
||
"gridPos": {"x": 16, "y": y, "w": 8, "h": 8},
|
||
"options": {"alertInstanceLabelFilter": "", "alertName": "", "dashboardAlerts": False, "groupBy": [],
|
||
"groupMode": "default", "maxItems": 20, "sortOrder": 1, "stateFilter":
|
||
{"firing": True, "pending": True, "noData": False, "normal": False, "error": True},
|
||
"viewMode": "list", "datasource": "Prometheus"}},
|
||
]
|
||
y += 4
|
||
panels += [
|
||
stat("RPS API", "messenger:http_requests:rate2m", "reqps", 0, y, decimals=1),
|
||
stat("p95 API (5m)", "messenger:http_latency_p95:rate5m", "s", 4, y,
|
||
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}]),
|
||
stat("Загрузка worker", "messenger:worker_saturation:ratio", "percentunit", 8, y,
|
||
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 0.9}]),
|
||
stat("Ожидают пул БД", "max(db_client_connection_pending_requests)", "short", 12, y,
|
||
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}]),
|
||
]
|
||
y += 4
|
||
# ------------------------------------------------------------------ API
|
||
panels.append(row("API (messenger-api, все реплики)", y)); y += 1
|
||
rate_count = "rate(%s_count{%s}[$__rate_interval])" % (HTTP, API)
|
||
panels += [
|
||
ts("RPS по маршруту и методу", [("sum by (http_request_method, http_route) (%s)" % rate_count,
|
||
"{{http_request_method}} {{http_route}}")], "reqps", 0, y,
|
||
desc="Counter = _count гистограммы http.server.request.duration; route — шаблон, не реальный UUID."),
|
||
ts("RPS по классу статуса", [(
|
||
'sum by (status_class) (label_replace(%s, "status_class", "${1}xx", "http_response_status_code", "(\\\\d)..")) ' % rate_count,
|
||
"{{status_class}}")], "reqps", 12, y, stack=True),
|
||
]
|
||
y += 8
|
||
bucket = "sum by (le) (rate(%s_bucket{%s}[$__rate_interval]))" % (HTTP, API)
|
||
panels += [
|
||
ts("Латентность API p50/p95/p99 (из histogram buckets)", [
|
||
(q(0.5, bucket), "p50", "A"), (q(0.95, bucket), "p95", "B"), (q(0.99, bucket), "p99", "C")], "s", 0, y,
|
||
desc="histogram_quantile по сумме buckets всех реплик; не среднее и не усреднение p99 реплик."),
|
||
ts("Доля ошибок 5xx", [
|
||
('(sum(rate(%s_count{%s,http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(%s)' % (HTTP, API, rate_count),
|
||
"5xx / все", "A")], "percentunit", 12, y,
|
||
desc="Числитель: 5xx; знаменатель: все бизнес-запросы (health исключён). 4xx не считаются отказом."),
|
||
]
|
||
y += 8
|
||
panels += [
|
||
ts("p95 по маршруту", [(q(0.95, "sum by (le, http_route) (rate(%s_bucket{%s}[$__rate_interval]))" % (HTTP, API)),
|
||
"{{http_route}}")], "s", 0, y),
|
||
ts("Запросы в обработке", [("sum by (instance) (http_server_active_requests{%s})" % API, "{{instance}}")],
|
||
"short", 12, y),
|
||
]
|
||
y += 8
|
||
# ------------------------------------------------------------------ worker and queue
|
||
panels.append(row("Очередь и worker", y)); y += 1
|
||
jb = "sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"succeeded\"}[$__rate_interval]))"
|
||
rb = "sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))"
|
||
panels += [
|
||
ts("Jobs по исходу (в сек.)", [("sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))",
|
||
"{{outcome}}")], "ops", 0, y, stack=True,
|
||
desc="succeeded / retry / failed / noop / lost / dlq / requeued; job_type=attachment.process.v1"),
|
||
ts("Длительность job (succeeded) p50/p95/p99", [(q(0.5, jb), "p50", "A"), (q(0.95, jb), "p95", "B"),
|
||
(q(0.99, jb), "p99", "C")], "s", 12, y),
|
||
]
|
||
y += 8
|
||
panels += [
|
||
ts("Глубина очереди (из PostgreSQL)", [("max by (queue, state) (messenger_queue_depth)", "{{queue}} {{state}}")],
|
||
"short", 0, y, desc="max() по репликам API: каждая публикует одно и то же значение из БД."),
|
||
ts("Возраст старейшей job", [('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})', "oldest age")],
|
||
"s", 12, y, desc="От durable acceptance (COMMIT job) до сейчас; это backlog, не processing latency.",
|
||
extra={"custom": {"lineWidth": 2, "fillOpacity": 10, "showPoints": "never",
|
||
"thresholdsStyle": {"mode": "line"}},
|
||
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None},
|
||
{"color": "red", "value": 60}]}}),
|
||
]
|
||
y += 8
|
||
panels += [
|
||
ts("Время до ready p50/p95/p99", [(q(0.5, rb), "p50", "A"), (q(0.95, rb), "p95", "B"), (q(0.99, rb), "p99", "C")],
|
||
"s", 0, y, desc="messenger.attachment.time_to_ready: от 202 (COMMIT job) до COMMIT ready."),
|
||
ts("Насыщение worker: занято / ёмкость", [
|
||
("sum(messenger_worker_busy)", "busy", "A"), ("sum(messenger_worker_capacity)", "capacity", "B"),
|
||
("messenger:worker_saturation:ratio", "saturation", "C")], "short", 12, y),
|
||
]
|
||
y += 8
|
||
# ------------------------------------------------------------------ resources
|
||
panels.append(row("Ресурсы: пул БД, CPU, RAM", y)); y += 1
|
||
panels += [
|
||
ts("Пул БД: used / idle / waiting", [
|
||
("sum by (service_name, db_client_connection_state) (db_client_connection_count)",
|
||
"{{service_name}} {{db_client_connection_state}}", "A"),
|
||
("sum by (service_name) (db_client_connection_pending_requests)", "{{service_name}} waiting", "B"),
|
||
("sum by (service_name) (db_client_connection_max)", "{{service_name}} max", "C")], "short", 0, y),
|
||
ts("CPU процессов (ядра)", [
|
||
("sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))",
|
||
"{{service_name}} {{instance}}", "A"),
|
||
("rate(otelcol_process_cpu_seconds_total[$__rate_interval])", "otel-collector", "B")], "percentunit", 12, y,
|
||
desc="worker: self + завершённые дочерние процессы обработки изображений."),
|
||
]
|
||
y += 8
|
||
panels += [
|
||
ts("RSS процессов", [("process_memory_usage_bytes", "{{service_name}} {{instance}}", "A"),
|
||
("otelcol_process_memory_rss_bytes", "otel-collector", "B")], "bytes", 0, y),
|
||
ts("Telemetry pipeline: принято / отправлено / ошибки (items/s)", [
|
||
("sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval]))", "accepted spans", "A"),
|
||
("sum(rate(otelcol_receiver_accepted_log_records_total[$__rate_interval]))", "accepted log records", "E"),
|
||
("sum(rate(otelcol_receiver_accepted_metric_points_total[$__rate_interval]))", "accepted metric points", "F"),
|
||
("sum by (exporter) (rate({__name__=~\"otelcol_exporter_sent_(spans|log_records|metric_points)_total\"}[$__rate_interval]))",
|
||
"sent {{exporter}}", "B"),
|
||
# Separate queries: rate() drops __name__, so a regex over several failed_* counters collides once they all exist.
|
||
("sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval])) or vector(0)", "send failed spans", "C"),
|
||
("sum(rate(otelcol_exporter_enqueue_failed_spans_total[$__rate_interval])) or vector(0)", "dropped spans (queue full)", "G"),
|
||
("sum(rate(otelcol_exporter_send_failed_log_records_total[$__rate_interval])) or vector(0)", "send failed logs", "H"),
|
||
("sum(rate(otelcol_exporter_enqueue_failed_log_records_total[$__rate_interval])) or vector(0)", "dropped logs (queue full)", "I"),
|
||
("sum by (exporter) (otelcol_exporter_queue_size)", "queue {{exporter}}", "D")], "short", 12, y),
|
||
]
|
||
y += 8
|
||
# ------------------------------------------------------------------ logs and traces
|
||
panels.append(row("Логи и трассы", y)); y += 1
|
||
panels += [
|
||
{"id": next(_ids), "type": "logs", "title": "Логи API и worker (warning/error)", "datasource": LOKI,
|
||
"gridPos": {"x": 0, "y": y, "w": 12, "h": 10},
|
||
"options": {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True},
|
||
"targets": [{"datasource": LOKI, "refId": "A",
|
||
"expr": '{service_name=~"messenger-api|messenger-worker"} | severity_text=~"WARN.*|ERROR"'}]},
|
||
{"id": next(_ids), "type": "table", "title": "Трассы с ошибками или медленнее 500 ms (TraceQL)", "datasource": TEMPO,
|
||
"gridPos": {"x": 12, "y": y, "w": 12, "h": 10},
|
||
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20,
|
||
"query": '{resource.service.name=~"messenger-.*" && (status=error || duration>500ms)}'}]},
|
||
]
|
||
y += 10
|
||
|
||
dashboard = {
|
||
"uid": "messenger-lab6", "title": "Messenger — наблюдаемость (lab6)", "tags": ["messenger", "lab6"],
|
||
"timezone": "browser", "schemaVersion": 39, "version": 1, "editable": False, "refresh": "10s",
|
||
"time": {"from": "now-30m", "to": "now"}, "graphTooltip": 1, "panels": panels,
|
||
"annotations": {"list": [{"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"},
|
||
"enable": True, "hide": True, "name": "Annotations & Alerts", "type": "dashboard"}]},
|
||
"templating": {"list": []},
|
||
}
|
||
OUT.write_text(json.dumps(dashboard, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
print("wrote %s (%d panels)" % (OUT.relative_to(OUT.parents[3]), len(panels)))
|
||
|
||
# Human-readable list of every panel: datasource, query and unit (kept in sync with the JSON by this script).
|
||
UNITS = {"reqps": "запросы/с", "percentunit": "доля (0..1)", "s": "секунды", "ops": "операции/с", "short": "штуки",
|
||
"bytes": "байты"}
|
||
DOC = OUT.parents[3] / "docs/dashboard.md"
|
||
lines = ["# Панели dashboard «%s»" % dashboard["title"], "",
|
||
"Сгенерировано `scripts/gen_dashboard.py` вместе с `infra/grafana/dashboards/messenger.json`. "
|
||
"`$__rate_interval` подставляет Grafana (не меньше 4 × scrape/export interval).", ""]
|
||
for panel in panels:
|
||
if panel["type"] == "row":
|
||
lines += ["## " + panel["title"], ""]
|
||
continue
|
||
unit = panel.get("fieldConfig", {}).get("defaults", {}).get("unit", "")
|
||
lines.append("**%s** (%s%s)" % (panel["title"], panel["type"], ", единица: " + UNITS.get(unit, unit) if unit else ""))
|
||
if panel.get("description"):
|
||
lines.append("— " + panel["description"])
|
||
lines.append("")
|
||
if panel["type"] == "alertlist":
|
||
lines += ["Правила Prometheus в состоянии pending/firing (`infra/prometheus/rules/messenger.yml`).", ""]
|
||
for t in panel.get("targets", []):
|
||
query = t.get("expr") or t.get("query")
|
||
legend = t.get("legendFormat", "")
|
||
lines += ["```%s" % ("promql" if t["datasource"]["type"] == "prometheus" else ""),
|
||
query + (" # " + legend if legend and legend != panel["title"] else ""), "```"]
|
||
lines.append("")
|
||
DOC.write_text("\n".join(lines), encoding="utf-8")
|
||
print("wrote docs/dashboard.md")
|