Перенесена реализация lab6; бизнес-API без изменений, транспорт — HTTPS. - Внешний HTTPS (public-ca) на 127.0.0.1:8443; HSTS, TLS >= 1.2. - Обязательное mTLS LB -> каждая API-реплика: API проверяет цепочку internal-ca и SAN клиента (allowlist: lb); health только на 127.0.0.1. - TLS с проверкой сервера ко всем зависимостям: PostgreSQL verify-full (pg_hba только hostssl), HTTPS S3, AMQPS, OTLP/HTTPS с client cert, mTLS Collector -> Prometheus/Loki/Tempo, Grafana -> datasources, scrape. - OpenBao 2.7.2 не в dev-mode: Raft на volume, init/unseal оператором, auth/cert с ролью и policy на сервис, TTL 15 мин + renew/re-login. - Отдельные роли PostgreSQL с грантами по таблицам; разные ключи S3 и пользователи брокера выдаются из OpenBao. - Argon2id (m=19 MiB, t=2, p=1, замер на стенде) и rehash при входе. - Случайный X-Instance-Id; hardening контейнеров (non-root, cap_drop ALL, no-new-privileges, read-only, лимиты); TCP_NODELAY на сокете API. - Модель угроз, trust matrix, отрицательные тесты TLS/mTLS, API, OpenBao, hardening; evidence, в т.ч. прогон из чистого clone.
324 lines
23 KiB
Python
Executable File
324 lines
23 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""Генератор импортируемого dashboard Grafana: infra/grafana/dashboards/messenger.json.
|
||
|
||
JSON лежит в Git и подхватывается provisioning-ом; этот скрипт — воспроизводимый способ его
|
||
изменить (python3 scripts/build_dashboard.py). Только стандартная библиотека.
|
||
Все запросы ссылаются на реальные имена метрик (см. REPORT.md «Схема сигналов»).
|
||
Перцентили — histogram_quantile по sum by (le) buckets всех реплик, не среднее p95.
|
||
"""
|
||
import json
|
||
from pathlib import Path
|
||
|
||
OUT = Path(__file__).resolve().parents[1] / "infra/grafana/dashboards/messenger.json"
|
||
PROM = {"type": "prometheus", "uid": "prometheus"}
|
||
LOKI = {"type": "loki", "uid": "loki"}
|
||
TEMPO = {"type": "tempo", "uid": "tempo"}
|
||
API = 'job="messenger-api", instance=~"$instance"'
|
||
BUSINESS = API + ', http_route!~"/health/.*|unmatched"'
|
||
|
||
_panels = []
|
||
_id = [0]
|
||
_pos = {"x": 0, "y": 0, "h": 0}
|
||
|
||
|
||
def _next_id():
|
||
_id[0] += 1
|
||
return _id[0]
|
||
|
||
|
||
def place(w, h):
|
||
if _pos["x"] + w > 24:
|
||
_pos["x"] = 0
|
||
_pos["y"] += _pos["h"]
|
||
_pos["h"] = 0
|
||
g = {"x": _pos["x"], "y": _pos["y"], "w": w, "h": h}
|
||
_pos["x"] += w
|
||
_pos["h"] = max(_pos["h"], h)
|
||
return g
|
||
|
||
|
||
def row(title):
|
||
_pos["x"] = 0
|
||
_pos["y"] += _pos["h"]
|
||
_pos["h"] = 0
|
||
_panels.append({"type": "row", "id": _next_id(), "title": title, "collapsed": False,
|
||
"gridPos": {"x": 0, "y": _pos["y"], "w": 24, "h": 1}, "panels": []})
|
||
_pos["y"] += 1
|
||
|
||
|
||
def target(expr, legend="", ref="A", exemplar=False, instant=False):
|
||
t = {"datasource": PROM, "expr": expr, "legendFormat": legend or "__auto", "refId": ref,
|
||
"range": not instant, "instant": instant}
|
||
if exemplar:
|
||
t["exemplar"] = True
|
||
return t
|
||
|
||
|
||
def ts(title, targets, unit="short", w=8, h=8, description="", stack=False, min0=True, thresholds=None):
|
||
refs = "ABCDEFGHIJ"
|
||
tg = [target(*t[:2], ref=refs[i], exemplar=(len(t) > 2 and t[2])) for i, t in enumerate(targets)]
|
||
p = {"type": "timeseries", "id": _next_id(), "title": title, "description": description,
|
||
"datasource": PROM, "gridPos": place(w, h), "targets": tg,
|
||
"fieldConfig": {"defaults": {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10,
|
||
"showPoints": "never",
|
||
"stacking": {"mode": "normal" if stack else "none"}},
|
||
**({"min": 0} if min0 else {})},
|
||
"overrides": []},
|
||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]},
|
||
"tooltip": {"mode": "multi", "sort": "desc"}}}
|
||
if thresholds:
|
||
p["fieldConfig"]["defaults"]["thresholds"] = {"mode": "absolute", "steps": thresholds}
|
||
p["fieldConfig"]["defaults"]["custom"]["thresholdsStyle"] = {"mode": "line+area"}
|
||
_panels.append(p)
|
||
|
||
|
||
def stat(title, expr, unit="short", w=3, h=4, steps=None, description="", decimals=None):
|
||
p = {"type": "stat", "id": _next_id(), "title": title, "description": description, "datasource": PROM,
|
||
"gridPos": place(w, h), "targets": [target(expr, "", instant=True)],
|
||
"fieldConfig": {"defaults": {"unit": unit, "thresholds": {"mode": "absolute", "steps": steps or [
|
||
{"color": "green", "value": None}]}}, "overrides": []},
|
||
"options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
|
||
"colorMode": "background", "graphMode": "area", "textMode": "value"}}
|
||
if decimals is not None:
|
||
p["fieldConfig"]["defaults"]["decimals"] = decimals
|
||
_panels.append(p)
|
||
|
||
|
||
def build():
|
||
# ------------------------------------------------------------------ обзор
|
||
row("Обзор и SLO")
|
||
stat("RPS бизнес-запросов", "messenger:http_requests:rate2m", "reqps",
|
||
description="sum(rate(http_server_request_duration_seconds_count{job=messenger-api, без /health}[2m]))")
|
||
stat("5xx, доля (2m)", "messenger:http_error_ratio:rate2m", "percentunit", decimals=2,
|
||
steps=[{"color": "green", "value": None}, {"color": "orange", "value": 0.01}, {"color": "red", "value": 0.05}],
|
||
description="Числитель: 5xx бизнес-запросов; знаменатель: все бизнес-запросы (без /health, unmatched).")
|
||
stat("p95 API (5m)", "messenger:http_latency_seconds:p95_5m", "s", decimals=3,
|
||
steps=[{"color": "green", "value": None}, {"color": "orange", "value": 0.5}, {"color": "red", "value": 1}])
|
||
stat("SLO доступности, 7d", "messenger:slo_availability:ratio_7d", "percentunit", decimals=3,
|
||
steps=[{"color": "red", "value": None}, {"color": "green", "value": 0.995}],
|
||
description="SLO: 99.5 % бизнес-запросов без 5xx за скользящие 7 дней.")
|
||
stat("Остаток error budget, 7d", "messenger:slo_availability:error_budget_remaining_7d", "percentunit",
|
||
decimals=1, steps=[{"color": "red", "value": None}, {"color": "orange", "value": 0.25},
|
||
{"color": "green", "value": 0.5}])
|
||
stat("Time to ready p95", "messenger:time_to_ready_seconds:p95_5m", "s", decimals=2,
|
||
steps=[{"color": "green", "value": None}, {"color": "orange", "value": 10}, {"color": "red", "value": 30}],
|
||
description="От durable acceptance (COMMIT enqueue) до COMMIT ready. SLO: 99 % не дольше 30 с.")
|
||
stat("Ready lag очереди", "max(messenger_jobs_ready_lag_seconds) or vector(0)", "s", decimals=0,
|
||
steps=[{"color": "green", "value": None}, {"color": "orange", "value": 10}, {"color": "red", "value": 30}],
|
||
description="Сколько ждёт worker самое старое созревшее задание (PostgreSQL jobs).")
|
||
stat("Alerts firing", 'count(ALERTS{alertstate="firing"}) or vector(0)', "short",
|
||
steps=[{"color": "green", "value": None}, {"color": "red", "value": 1}])
|
||
_panels.append({
|
||
"type": "state-timeline", "id": _next_id(), "title": "История alerts (pending/firing -> resolved)",
|
||
"datasource": PROM, "gridPos": place(24, 6),
|
||
"targets": [target('max by (alertname, source, alertstate) (ALERTS)', "{{alertname}} {{source}} {{alertstate}}")],
|
||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"},
|
||
"thresholds": {"mode": "absolute", "steps": [
|
||
{"color": "transparent", "value": None}, {"color": "red", "value": 1}]}},
|
||
"overrides": []},
|
||
"options": {"showValue": "never", "mergeValues": True, "rowHeight": 0.8,
|
||
"legend": {"showLegend": False}}})
|
||
|
||
# ------------------------------------------------------------------ API
|
||
row("API: трафик, ошибки, задержка")
|
||
ts("RPS по маршруту", [("sum by (http_route) (rate(http_server_request_duration_seconds_count{%s}[1m]))" % API,
|
||
"{{http_route}}")], "reqps", stack=True)
|
||
ts("Ответы по классу статуса", [(
|
||
"sum by (http_response_status_class) (rate(http_server_request_duration_seconds_count{%s}[1m]))" % BUSINESS,
|
||
"{{http_response_status_class}}")], "reqps", stack=True)
|
||
ts("Доля 5xx: API и LB", [
|
||
("messenger:http_error_ratio:rate2m", "API (без /health)"),
|
||
("messenger:lb_error_ratio:rate2m", "HAProxy fe_api")], "percentunit",
|
||
thresholds=[{"color": "transparent", "value": None}, {"color": "red", "value": 0.05}],
|
||
description="Если БД недоступна, реплики не ready и 503 отдаёт LB: эти запросы видны только в метрике LB.")
|
||
ts("Задержка API p50/p95/p99 (из histogram)", [
|
||
("histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p50", True),
|
||
("histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p95", True),
|
||
("histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p99", True)],
|
||
"s", w=12, description="Точки-exemplars ведут в трассу Tempo конкретного запроса.")
|
||
ts("p95 по маршруту", [(
|
||
"histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{%s}[5m])))" % BUSINESS,
|
||
"{{http_route}}")], "s", w=12)
|
||
ts("5xx по причине (error.type)", [(
|
||
'sum by (error_type, http_route) (rate(http_server_request_duration_seconds_count{%s, http_response_status_class="5xx"}[1m]))' % API,
|
||
"{{error_type}} {{http_route}}")], "reqps", stack=True)
|
||
ts("Запросы в обработке (saturation API)", [
|
||
("sum by (instance) (http_server_active_requests{%s})" % API, "{{instance}}")], "short")
|
||
ts("HAProxy: UP-реплики и очередь LB", [
|
||
('haproxy_backend_active_servers{proxy="be_api"}', "UP реплик"),
|
||
('haproxy_backend_current_queue{proxy="be_api"}', "очередь LB")], "short")
|
||
|
||
# ------------------------------------------------------------------ worker
|
||
row("Worker и очередь заданий")
|
||
ts("Задания по исходу (в минуту)", [(
|
||
"sum by (messenger_job_outcome) (increase(messenger_jobs_processed_total[1m]))", "{{messenger_job_outcome}}")],
|
||
"short", stack=True, description="ready / retry / failed / duplicate / lease_lost / rejected / requeued")
|
||
ts("Длительность обработки job p50/p95/p99", [
|
||
('histogram_quantile(0.50, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p50", True),
|
||
('histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p95", True),
|
||
('histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p99", True)],
|
||
"s")
|
||
ts("Time to ready p50/p95/p99", [
|
||
('histogram_quantile(0.50, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p50", True),
|
||
('histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p95", True),
|
||
('histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p99", True)],
|
||
"s", description="Backlog-ожидание + обработка: время от 202 (COMMIT) до ready.")
|
||
ts("Глубина очереди: PostgreSQL и RabbitMQ", [
|
||
('max by (messenger_job_status) (messenger_jobs_count{messenger_job_status=~"queued|processing"})', "jobs {{messenger_job_status}} (БД)"),
|
||
('max(messenger_outbox_pending)', "outbox не опубликовано"),
|
||
('sum(rabbitmq_detailed_queue_messages_ready{queue="attachments.process.v1"})', "RabbitMQ ready"),
|
||
('sum(rabbitmq_detailed_queue_messages_unacked{queue="attachments.process.v1"})', "RabbitMQ unacked"),
|
||
('sum(rabbitmq_detailed_queue_messages{queue="attachments.dlq"})', "DLQ")], "short")
|
||
ts("Возраст старейшего задания и lag", [
|
||
('max by (messenger_job_status) (messenger_jobs_oldest_age_seconds)', "oldest {{messenger_job_status}}"),
|
||
("max(messenger_jobs_ready_lag_seconds)", "ready lag"),
|
||
("max(messenger_outbox_oldest_age_seconds)", "outbox oldest")], "s",
|
||
thresholds=[{"color": "transparent", "value": None}, {"color": "red", "value": 30}])
|
||
ts("Ожидание в очереди p95 (до claim)", [
|
||
("histogram_quantile(0.95, sum by (le) (rate(messenger_job_queue_wait_seconds_bucket[5m])))", "p95 queue wait"),
|
||
("histogram_quantile(0.95, sum by (le) (rate(messenger_outbox_publish_delay_seconds_bucket[5m])))", "p95 outbox -> broker")],
|
||
"s", description="Backlog (сколько ждут) отдельно от processing latency (сколько обрабатываются).")
|
||
ts("Saturation worker: занятость потребителей", [
|
||
("sum(rate(messenger_worker_busy_time_seconds_total[1m])) / sum(last_over_time(messenger_worker_capacity[30s]))", "utilization (busy_time/capacity)"),
|
||
("sum(messenger_worker_busy) / sum(last_over_time(messenger_worker_capacity[30s]))", "busy/capacity (мгновенно)")],
|
||
"percentunit", description="1.0 = все WORKER_CONCURRENCY потоков всех реплик заняты; очередь растёт быстрее обработки.")
|
||
ts("Relay: публикации outbox", [
|
||
("sum by (outcome) (rate(messenger_outbox_published_total[1m]))", "{{outcome}}")], "short")
|
||
|
||
# ------------------------------------------------------------------ DB / S3
|
||
row("PostgreSQL и S3")
|
||
ts("Пул соединений API: used / idle / max", [
|
||
('sum by (instance, db_client_connection_state) (db_client_connection_count{%s})' % API,
|
||
"{{instance}} {{db_client_connection_state}}"),
|
||
('max(db_client_connection_max{%s})' % API, "max на реплику")], "short")
|
||
ts("Ожидающие соединение и таймауты пула", [
|
||
('sum by (instance) (db_client_connection_pending_requests{%s})' % API, "pending {{instance}}"),
|
||
('sum by (instance) (increase(db_client_connection_timeouts_total{%s}[1m]))' % API, "timeouts/мин {{instance}}")],
|
||
"short")
|
||
ts("Ожидание соединения пула p95", [
|
||
('histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket{%s}[1m])))' % API, "p95 wait")],
|
||
"s")
|
||
ts("Операции БД p95 по операции (API)", [
|
||
('histogram_quantile(0.95, sum by (le, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{%s}[1m])))' % API,
|
||
"{{db_operation_name}}")], "s", w=12)
|
||
ts("Операции БД worker/relay p95", [
|
||
('histogram_quantile(0.95, sum by (le, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{job=~"messenger-worker|messenger-relay"}[5m])))',
|
||
"{{db_operation_name}}")], "s", w=12)
|
||
ts("Ошибки операций БД", [
|
||
('sum by (job, db_operation_name, error_type) (rate(db_client_operation_duration_seconds_count{error_type!=""}[1m]))',
|
||
"{{job}} {{db_operation_name}} {{error_type}}")], "short")
|
||
ts("S3 p95 по операции", [
|
||
('histogram_quantile(0.95, sum by (le, messenger_s3_operation) (rate(messenger_s3_operation_duration_seconds_bucket[5m])))',
|
||
"{{messenger_s3_operation}}")], "s")
|
||
ts("S3 ошибки", [
|
||
('sum by (job, messenger_s3_operation, outcome) (rate(messenger_s3_operation_duration_seconds_count{outcome!="ok"}[1m]))',
|
||
"{{job}} {{messenger_s3_operation}} {{outcome}}")], "short")
|
||
|
||
# ------------------------------------------------------------------ ресурсы
|
||
row("Ресурсы контейнеров (CPU/RAM)")
|
||
ts("CPU, ядер (cgroup контейнера)", [
|
||
("sum by (job, instance) (rate(container_cpu_time_seconds_total[1m]))", "{{job}} {{instance}}")], "short", w=12)
|
||
ts("CPU saturation: доля лимита", [
|
||
("rate(container_cpu_time_seconds_total[1m]) / on(job, instance) container_cpu_limit", "{{job}} {{instance}}")],
|
||
"percentunit", w=12)
|
||
ts("Память контейнеров приложений", [
|
||
("container_memory_usage_bytes", "{{job}} {{instance}}")], "bytes", w=12)
|
||
ts("Память: доля лимита", [
|
||
("container_memory_usage_bytes / on(job, instance) container_memory_limit_bytes", "{{job}} {{instance}}")],
|
||
"percentunit", w=12)
|
||
ts("Память telemetry-стека (RSS)", [
|
||
('process_resident_memory_bytes{job=~"prometheus|loki|tempo"}', "{{job}}"),
|
||
('{__name__=~"otelcol_process_memory_rss(_bytes)?"}', "otel-collector RSS (вкл. страницы бинарника)"),
|
||
('otelcol_process_runtime_heap_alloc_bytes', "otel-collector Go heap"),
|
||
('rabbitmq_process_resident_memory_bytes', "rabbitmq")], "bytes", w=12)
|
||
ts("CPU telemetry-стека", [
|
||
('rate(process_cpu_seconds_total{job=~"prometheus|loki|tempo"}[1m])', "{{job}}"),
|
||
('rate({__name__=~"otelcol_process_cpu_seconds(_total)?"}[1m])', "otel-collector")], "short", w=12)
|
||
|
||
# ------------------------------------------------------------------ telemetry
|
||
row("Конвейер телеметрии (потери)")
|
||
ts("Collector: принято / отправлено / потеряно (spans)", [
|
||
('sum(rate({__name__=~"otelcol_receiver_accepted_spans(_total)?"}[1m]))', "принято"),
|
||
('sum(rate({__name__=~"otelcol_exporter_sent_spans(_total)?"}[1m]))', "отправлено в Tempo"),
|
||
('sum(rate({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[1m]))', "ошибка отправки"),
|
||
('sum(rate({__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?"}[1m]))', "очередь полна (потеря)"),
|
||
('sum(rate({__name__=~"otelcol_receiver_refused_spans(_total)?"}[1m]))', "отказ приёма (memory_limiter)")], "short")
|
||
ts("Collector: логи", [
|
||
('sum(rate({__name__=~"otelcol_receiver_accepted_log_records(_total)?"}[1m]))', "принято"),
|
||
('sum(rate({__name__=~"otelcol_exporter_sent_log_records(_total)?"}[1m]))', "отправлено в Loki"),
|
||
('sum(rate({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}[1m]))', "ошибка отправки"),
|
||
('sum(rate({__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?"}[1m]))', "очередь полна (потеря)")], "short")
|
||
ts("Collector: очередь экспортёров", [
|
||
("max by (exporter) (otelcol_exporter_queue_size)", "size {{exporter}}"),
|
||
("max by (exporter) (otelcol_exporter_queue_capacity)", "capacity {{exporter}}")], "short")
|
||
ts("SDK приложений: потери на стороне процесса", [
|
||
('sum by (job, error_type) (rate(otel_sdk_processor_span_processed_total{error_type!=""}[1m]))', "spans dropped {{job}} {{error_type}}"),
|
||
('sum by (job, error_type) (rate(otel_sdk_processor_log_processed_total{error_type!=""}[1m]))', "logs dropped {{job}} {{error_type}}"),
|
||
('sum by (job, error_type) (rate(otel_sdk_exporter_span_exported_total{error_type!=""}[1m]))', "span export failed {{job}} {{error_type}}")],
|
||
"short", w=12)
|
||
ts("SDK: заполнение очереди BatchSpanProcessor", [
|
||
("max by (job, instance) (otel_sdk_processor_span_queue_size)", "{{job}} {{instance}}"),
|
||
("max(otel_sdk_processor_span_queue_capacity)", "capacity")], "short", w=12)
|
||
|
||
# ------------------------------------------------------------------ logs / traces
|
||
row("Логи и трассы")
|
||
_panels.append({
|
||
"type": "logs", "id": _next_id(), "title": "Логи WARN/ERROR приложений (Loki)", "datasource": LOKI,
|
||
"gridPos": place(24, 10),
|
||
"targets": [{"datasource": LOKI, "refId": "A", "queryType": "range",
|
||
"expr": '{service_name=~"messenger-.+"} | severity_text=~"WARN.*|ERROR.*|CRITICAL"'}],
|
||
"options": {"showTime": True, "wrapLogMessage": True, "enableLogDetails": True, "sortOrder": "Descending",
|
||
"dedupStrategy": "none", "prettifyLogMessage": False}})
|
||
_panels.append({
|
||
"type": "logs", "id": _next_id(), "title": "HTTP 5xx (event=http.request)", "datasource": LOKI,
|
||
"gridPos": place(12, 10),
|
||
"targets": [{"datasource": LOKI, "refId": "A", "queryType": "range",
|
||
"expr": '{service_name="messenger-api"} | event="http.request" | http_response_status_code >= 500'}],
|
||
"options": {"showTime": True, "wrapLogMessage": True, "enableLogDetails": True, "sortOrder": "Descending"}})
|
||
_panels.append({
|
||
"type": "table", "id": _next_id(), "title": "Последние трассы обработки заданий (Tempo)", "datasource": TEMPO,
|
||
"gridPos": place(12, 10),
|
||
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces",
|
||
"query": '{ resource.service.name = "messenger-worker" && name = "attachments.process.v1 process" }'}]})
|
||
_panels.append({
|
||
"type": "table", "id": _next_id(), "title": "Медленные запросы API > 1 s (Tempo)", "datasource": TEMPO,
|
||
"gridPos": place(12, 10),
|
||
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces",
|
||
"query": '{ resource.service.name = "messenger-api" && kind = server && duration > 1s }'}]})
|
||
_panels.append({
|
||
"type": "table", "id": _next_id(), "title": "Трассы с ошибкой (Tempo)", "datasource": TEMPO,
|
||
"gridPos": place(12, 10),
|
||
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces",
|
||
"query": '{ status = error }'}]})
|
||
|
||
return {
|
||
"uid": "messenger-overview",
|
||
"title": "Messenger — наблюдаемость (lab6)",
|
||
"description": "API, worker, очередь, PostgreSQL, S3, ресурсы, конвейер телеметрии. Provisioning из Git.",
|
||
"tags": ["messenger", "lab6", "otel"],
|
||
"timezone": "browser",
|
||
"editable": False,
|
||
"graphTooltip": 1,
|
||
"refresh": "10s",
|
||
"schemaVersion": 39,
|
||
"version": 1,
|
||
"time": {"from": "now-30m", "to": "now"},
|
||
"templating": {"list": [{
|
||
"name": "instance", "label": "API-реплика", "type": "query", "datasource": PROM,
|
||
"query": {"query": 'label_values(http_server_request_duration_seconds_count{job="messenger-api"}, instance)',
|
||
"refId": "PrometheusVariableQueryEditor-VariableQuery"},
|
||
"definition": 'label_values(http_server_request_duration_seconds_count{job="messenger-api"}, instance)',
|
||
"includeAll": True, "multi": True, "allValue": ".*", "current": {"text": "All", "value": "$__all"},
|
||
"refresh": 2, "sort": 1}]},
|
||
"annotations": {"list": [{
|
||
"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, "enable": True, "hide": True,
|
||
"iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard"}]},
|
||
"panels": _panels,
|
||
}
|
||
|
||
|
||
if __name__ == "__main__":
|
||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||
OUT.write_text(json.dumps(build(), ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
||
print("written %s (%d panels)" % (OUT.relative_to(OUT.parents[3]), len(_panels)))
|