Files
mtusi2026lab7/scripts/gen_dashboard.py
T

171 lines
11 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Генератор dashboard Grafana (infra/grafana/dashboards/messenger.json).
JSON в Git — то, что импортирует provisioning; этот скрипт лишь делает его
воспроизводимым: python3 scripts/gen_dashboard.py. Все percentile считаются
histogram_quantile по сумме buckets всех реплик (sum by le), не из средних.
"""
import json
import pathlib
PROM = {'type': 'prometheus', 'uid': 'prometheus'}
LOKI = {'type': 'loki', 'uid': 'loki'}
TEMPO = {'type': 'tempo', 'uid': 'tempo'}
API = 'job="messenger-api", http_route!~"/health/.*"'
panels = []
y = 0
pid = 0
def nid():
global pid
pid += 1
return pid
def row(title):
global y
panels.append({'type': 'row', 'title': title, 'id': nid(), 'collapsed': False,
'gridPos': {'h': 1, 'w': 24, 'x': 0, 'y': y}, 'panels': []})
y += 1
def ts(title, targets, unit, x, w=8, h=8, desc='', stack=False, minv=None, maxv=None):
p = {'type': 'timeseries', 'title': title, 'id': nid(), 'datasource': PROM, 'description': desc,
'gridPos': {'h': h, 'w': w, 'x': x, 'y': y},
'fieldConfig': {'defaults': {'unit': unit, 'custom': {'fillOpacity': 10, 'showPoints': 'never',
'stacking': {'mode': 'normal' if stack else 'none'}}},
'overrides': []},
'options': {'legend': {'displayMode': 'list', 'placement': 'bottom'}, 'tooltip': {'mode': 'multi'}},
'targets': [{'datasource': PROM, 'expr': e, 'legendFormat': l, 'refId': chr(65 + i),
'exemplar': True} for i, (e, l) in enumerate(targets)]}
if minv is not None:
p['fieldConfig']['defaults']['min'] = minv
if maxv is not None:
p['fieldConfig']['defaults']['max'] = maxv
panels.append(p)
def stat(title, expr, unit, x, w=4, h=4, desc='', thresholds=None):
steps = thresholds or [{'color': 'green', 'value': None}]
panels.append({'type': 'stat', 'title': title, 'id': nid(), 'datasource': PROM, 'description': desc,
'gridPos': {'h': h, 'w': w, 'x': x, 'y': y},
'fieldConfig': {'defaults': {'unit': unit, 'thresholds': {'mode': 'absolute', 'steps': steps}},
'overrides': []},
'options': {'reduceOptions': {'calcs': ['lastNotNull']}, 'colorMode': 'value'},
'targets': [{'datasource': PROM, 'expr': expr, 'refId': 'A'}]})
def q(expr, by='le'):
return lambda p: f'histogram_quantile({p}, sum by ({by}) (rate({expr}[$__rate_interval])))'
# ---------------- SLO / сводка ----------------
row('Сводка и SLO')
stat('RPS API (бизнес)', 'messenger:http_requests:rate1m', 'reqps', 0,
desc='sum(rate(http_server_request_duration_seconds_count)) без /health/*')
stat('Доля 5xx (1m)', 'messenger:http_error_ratio:rate1m', 'percentunit', 4,
desc='5xx / все бизнес-запросы (кроме /health/*). SLO: ≤ 0.5%',
thresholds=[{'color': 'green', 'value': None}, {'color': 'orange', 'value': 0.005}, {'color': 'red', 'value': 0.05}])
stat('Быстрее 250 ms (5m)', 'messenger:http_fast_ratio:rate5m', 'percentunit', 8,
desc='SLO задержки: ≥ 95% бизнес-запросов быстрее 250 ms',
thresholds=[{'color': 'red', 'value': None}, {'color': 'green', 'value': 0.95}])
stat('Ready быстрее 30 s (5m)', 'messenger:time_to_ready_fast_ratio:rate5m', 'percentunit', 12,
desc='SLO времени до ready: ≥ 95% вложений',
thresholds=[{'color': 'red', 'value': None}, {'color': 'green', 'value': 0.95}])
stat('Старейшая job', 'max(messenger_jobs_oldest_age_seconds)', 's', 16,
thresholds=[{'color': 'green', 'value': None}, {'color': 'orange', 'value': 30}, {'color': 'red', 'value': 60}])
stat('Alerts firing', 'count(ALERTS{alertstate="firing"}) or vector(0)', 'none', 20,
thresholds=[{'color': 'green', 'value': None}, {'color': 'red', 'value': 1}])
y += 4
# ---------------- API ----------------
row('API (messenger-api)')
hist = f'http_server_request_duration_seconds_bucket{{{API}}}'
ts('RPS по маршруту и классу ответа',
[(f'sum by (http_route, http_response_status_class) (rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))',
'{{http_route}} {{http_response_status_class}}')], 'reqps', 0, desc='Счётчик = _count histogram (одна метрика, не две)')
ts('Задержка API p50 / p95 / p99 (все реплики)',
[(q(hist)(0.5), 'p50'), (q(hist)(0.95), 'p95'), (q(hist)(0.99), 'p99')], 's', 8,
desc='histogram_quantile по сумме buckets всех реплик; среднее p99 реплик не используется')
ts('Доля ошибок 5xx и 4xx',
[('messenger:http_error_ratio:rate1m', '5xx / все'),
(f'sum(rate(http_server_request_duration_seconds_count{{{API}, http_response_status_class="4xx"}}[1m])) / messenger:http_requests:rate1m', '4xx / все (ошибки клиента)')],
'percentunit', 16, minv=0)
y += 8
ts('p95 по маршруту', [(f'histogram_quantile(0.95, sum by (le, http_route) (rate({hist}[$__rate_interval])))', '{{http_route}}')],
's', 0, w=12)
ts('Запросы в обработке (in-flight) по реплике',
[('sum by (instance) (http_server_active_requests{job="messenger-api"})', '{{instance}}')], 'none', 12, w=12)
y += 8
# ---------------- Worker / очередь ----------------
row('Worker и очередь')
ts('Jobs по исходу (rate)', [('sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))', '{{outcome}}')],
'ops', 0, desc='done / retry / failed / duplicate / not_due / requeue …')
jd = 'messenger_job_duration_seconds_bucket{outcome="done"}'
ts('Длительность обработки job p50 / p95 / p99', [(q(jd)(0.5), 'p50'), (q(jd)(0.95), 'p95'), (q(jd)(0.99), 'p99')], 's', 8)
tr = 'messenger_attachment_time_to_ready_seconds_bucket'
ts('Время до ready (202 → ready) p50 / p95', [(q(tr)(0.5), 'p50'), (q(tr)(0.95), 'p95')], 's', 16)
y += 8
ts('Очередь: jobs по состоянию (PostgreSQL)', [('sum by (state) (messenger_jobs_backlog)', '{{state}}'),
('sum(messenger_outbox_pending)', 'outbox не опубликовано')], 'none', 0)
ts('Возраст старейшей job / outbox', [('max(messenger_jobs_oldest_age_seconds)', 'job (queued/processing)'),
('max(messenger_outbox_oldest_age_seconds)', 'outbox')], 's', 8,
desc='Backlog (сколько ждёт) ≠ latency обработки; alert при > 60 s')
ts('RabbitMQ: сообщения в очередях',
[('sum by (queue) (rabbitmq_detailed_queue_messages_ready or rabbitmq_queue_messages_ready)', 'ready {{queue}}'),
('sum by (queue) (rabbitmq_detailed_queue_messages_unacked or rabbitmq_queue_messages_unacked)', 'unacked {{queue}}')],
'none', 16)
y += 8
ts('Saturation worker (busy / capacity)',
[('sum(messenger_worker_busy) / sum(messenger_worker_capacity)', 'занято слотов')], 'percentunit', 0, minv=0, maxv=1)
ts('Publish outbox → broker (confirm) p95',
[(q('messenger_outbox_publish_duration_seconds_bucket')(0.95), 'p95'),
('sum(rate(messenger_outbox_published_total[$__rate_interval]))', 'сообщений/с')], 's', 8)
ts('Пул БД: занято / ожидания',
[('sum by (job) (pgxpool_acquired_connections)', 'занято {{job}}'),
('sum by (job) (pgxpool_max_connections)', 'max {{job}}'),
('sum by (job) (rate(pgxpool_empty_acquire_total[$__rate_interval]))', 'ожиданий/с {{job}}')], 'none', 16,
desc='empty_acquire — acquire, которому пришлось ждать свободное соединение')
y += 8
# ---------------- Ресурсы ----------------
row('Ресурсы процессов и телеметрии')
ts('CPU процессов', [('sum by (job, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))', '{{job}} {{instance}}')],
'percentunit', 0, desc='доля одного ядра; лимиты контейнеров в compose.yaml')
ts('Память Go (used)', [('sum by (job, instance) (go_memory_used_bytes)', '{{job}} {{instance}}')], 'bytes', 8)
ts('Collector: принято / отправлено / потеряно spans',
[('sum(rate({__name__=~"otelcol_receiver_accepted_spans(_total)?"}[$__rate_interval]))', 'принято'),
('sum(rate({__name__=~"otelcol_exporter_sent_spans(_total)?"}[$__rate_interval]))', 'отправлено'),
('sum(rate({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[$__rate_interval]))', 'не доставлено'),
('sum(rate({__name__=~"otelcol_receiver_refused_spans(_total)?"}[$__rate_interval]))', 'отклонено (memory_limiter)'),
('sum(otelcol_exporter_queue_size)', 'очередь экспортёров')], 'none', 16)
y += 8
# ---------------- Логи и трассы ----------------
row('Логи и трассы')
panels.append({'type': 'logs', 'title': 'Логи API и worker (ошибки и предупреждения)', 'id': nid(), 'datasource': LOKI,
'gridPos': {'h': 10, 'w': 12, 'x': 0, 'y': y},
'options': {'showTime': True, 'wrapLogMessage': True, 'enableLogDetails': True, 'sortOrder': 'Descending'},
'targets': [{'datasource': LOKI, 'refId': 'A',
'expr': '{service_name=~"messenger-(api|worker|relay)"} | detected_level=~"warn|error|WARN|ERROR"'}]})
panels.append({'type': 'table', 'title': 'Трассы worker (последние)', 'id': nid(), 'datasource': TEMPO,
'gridPos': {'h': 10, 'w': 12, 'x': 12, 'y': y},
'targets': [{'datasource': TEMPO, 'refId': 'A', 'queryType': 'traceql', 'limit': 20,
'query': '{resource.service.name="messenger-worker" && name="attachment.process process"}'}]})
y += 10
dash = {
'uid': 'messenger-overview', 'title': 'Messenger — API, worker, очередь', 'tags': ['lab6', 'messenger'],
'timezone': 'browser', 'schemaVersion': 39, 'version': 1, 'refresh': '10s',
'time': {'from': 'now-30m', 'to': 'now'}, 'editable': False, 'graphTooltip': 1,
'panels': panels, 'templating': {'list': []}, 'annotations': {'list': [
{'name': 'Alerts', 'datasource': PROM, 'enable': True, 'iconColor': 'red',
'expr': 'ALERTS{alertstate="firing"}', 'titleFormat': '{{alertname}}', 'step': '10s'}]},
}
out = pathlib.Path(__file__).resolve().parent.parent / 'infra' / 'grafana' / 'dashboards' / 'messenger.json'
out.write_text(json.dumps(dash, ensure_ascii=False, indent=1) + '\n', encoding='utf-8')
print(out, len(panels), 'panels')