169 lines
11 KiB
Python
169 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""Генератор dashboard Grafana (infra/grafana/dashboards/messenger.json).
|
||
|
||
JSON в Git — то, что импортирует provisioning; этот скрипт лишь делает его
|
||
воспроизводимым: python3 scripts/gen_dashboard.py. Все percentile считаются
|
||
histogram_quantile по сумме buckets всех реплик (sum by le), не из средних.
|
||
"""
|
||
import json
|
||
import pathlib
|
||
|
||
PROM = {'type': 'prometheus', 'uid': 'prometheus'}
|
||
LOKI = {'type': 'loki', 'uid': 'loki'}
|
||
TEMPO = {'type': 'tempo', 'uid': 'tempo'}
|
||
API = 'job="messenger-api", http_route!~"/health/.*"'
|
||
|
||
panels = []
|
||
y = 0
|
||
pid = 0
|
||
|
||
|
||
def nid():
|
||
global pid
|
||
pid += 1
|
||
return pid
|
||
|
||
|
||
def row(title):
|
||
global y
|
||
panels.append({'type': 'row', 'title': title, 'id': nid(), 'collapsed': False,
|
||
'gridPos': {'h': 1, 'w': 24, 'x': 0, 'y': y}, 'panels': []})
|
||
y += 1
|
||
|
||
|
||
def ts(title, targets, unit, x, w=8, h=8, desc='', stack=False, minv=None):
|
||
p = {'type': 'timeseries', 'title': title, 'id': nid(), 'datasource': PROM, 'description': desc,
|
||
'gridPos': {'h': h, 'w': w, 'x': x, 'y': y},
|
||
'fieldConfig': {'defaults': {'unit': unit, 'custom': {'fillOpacity': 10, 'showPoints': 'never',
|
||
'stacking': {'mode': 'normal' if stack else 'none'}}},
|
||
'overrides': []},
|
||
'options': {'legend': {'displayMode': 'list', 'placement': 'bottom'}, 'tooltip': {'mode': 'multi'}},
|
||
'targets': [{'datasource': PROM, 'expr': e, 'legendFormat': l, 'refId': chr(65 + i),
|
||
'exemplar': True} for i, (e, l) in enumerate(targets)]}
|
||
if minv is not None:
|
||
p['fieldConfig']['defaults']['min'] = minv
|
||
panels.append(p)
|
||
|
||
|
||
def stat(title, expr, unit, x, w=4, h=4, desc='', thresholds=None):
|
||
steps = thresholds or [{'color': 'green', 'value': None}]
|
||
panels.append({'type': 'stat', 'title': title, 'id': nid(), 'datasource': PROM, 'description': desc,
|
||
'gridPos': {'h': h, 'w': w, 'x': x, 'y': y},
|
||
'fieldConfig': {'defaults': {'unit': unit, 'thresholds': {'mode': 'absolute', 'steps': steps}},
|
||
'overrides': []},
|
||
'options': {'reduceOptions': {'calcs': ['lastNotNull']}, 'colorMode': 'value'},
|
||
'targets': [{'datasource': PROM, 'expr': expr, 'refId': 'A'}]})
|
||
|
||
|
||
def q(expr, by='le'):
|
||
return lambda p: f'histogram_quantile({p}, sum by ({by}) (rate({expr}[$__rate_interval])))'
|
||
|
||
|
||
# ---------------- SLO / сводка ----------------
|
||
row('Сводка и SLO')
|
||
stat('RPS API (бизнес)', 'messenger:http_requests:rate1m', 'reqps', 0,
|
||
desc='sum(rate(http_server_request_duration_seconds_count)) без /health/*')
|
||
stat('Доля 5xx (1m)', 'messenger:http_error_ratio:rate1m', 'percentunit', 4,
|
||
desc='5xx / все бизнес-запросы (кроме /health/*). SLO: ≤ 0.5%',
|
||
thresholds=[{'color': 'green', 'value': None}, {'color': 'orange', 'value': 0.005}, {'color': 'red', 'value': 0.05}])
|
||
stat('Быстрее 250 ms (5m)', 'messenger:http_fast_ratio:rate5m', 'percentunit', 8,
|
||
desc='SLO задержки: ≥ 95% бизнес-запросов быстрее 250 ms',
|
||
thresholds=[{'color': 'red', 'value': None}, {'color': 'green', 'value': 0.95}])
|
||
stat('Ready быстрее 30 s (5m)', 'messenger:time_to_ready_fast_ratio:rate5m', 'percentunit', 12,
|
||
desc='SLO времени до ready: ≥ 95% вложений',
|
||
thresholds=[{'color': 'red', 'value': None}, {'color': 'green', 'value': 0.95}])
|
||
stat('Старейшая job', 'max(messenger_jobs_oldest_age_seconds)', 's', 16,
|
||
thresholds=[{'color': 'green', 'value': None}, {'color': 'orange', 'value': 30}, {'color': 'red', 'value': 60}])
|
||
stat('Alerts firing', 'count(ALERTS{alertstate="firing"}) or vector(0)', 'none', 20,
|
||
thresholds=[{'color': 'green', 'value': None}, {'color': 'red', 'value': 1}])
|
||
y += 4
|
||
|
||
# ---------------- API ----------------
|
||
row('API (messenger-api)')
|
||
hist = f'http_server_request_duration_seconds_bucket{{{API}}}'
|
||
ts('RPS по маршруту и классу ответа',
|
||
[(f'sum by (http_route, http_response_status_class) (rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))',
|
||
'{{http_route}} {{http_response_status_class}}')], 'reqps', 0, desc='Счётчик = _count histogram (одна метрика, не две)')
|
||
ts('Задержка API p50 / p95 / p99 (все реплики)',
|
||
[(q(hist)(0.5), 'p50'), (q(hist)(0.95), 'p95'), (q(hist)(0.99), 'p99')], 's', 8,
|
||
desc='histogram_quantile по сумме buckets всех реплик; среднее p99 реплик не используется')
|
||
ts('Доля ошибок 5xx и 4xx',
|
||
[('messenger:http_error_ratio:rate1m', '5xx / все'),
|
||
(f'sum(rate(http_server_request_duration_seconds_count{{{API}, http_response_status_class="4xx"}}[1m])) / messenger:http_requests:rate1m', '4xx / все (ошибки клиента)')],
|
||
'percentunit', 16, minv=0)
|
||
y += 8
|
||
ts('p95 по маршруту', [(f'histogram_quantile(0.95, sum by (le, http_route) (rate({hist}[$__rate_interval])))', '{{http_route}}')],
|
||
's', 0, w=12)
|
||
ts('Запросы в обработке (in-flight) по реплике',
|
||
[('sum by (instance) (http_server_active_requests{job="messenger-api"})', '{{instance}}')], 'none', 12, w=12)
|
||
y += 8
|
||
|
||
# ---------------- Worker / очередь ----------------
|
||
row('Worker и очередь')
|
||
ts('Jobs по исходу (rate)', [('sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))', '{{outcome}}')],
|
||
'ops', 0, desc='done / retry / failed / duplicate / not_due / requeue …')
|
||
jd = 'messenger_job_duration_seconds_bucket{outcome="done"}'
|
||
ts('Длительность обработки job p50 / p95 / p99', [(q(jd)(0.5), 'p50'), (q(jd)(0.95), 'p95'), (q(jd)(0.99), 'p99')], 's', 8)
|
||
tr = 'messenger_attachment_time_to_ready_seconds_bucket'
|
||
ts('Время до ready (202 → ready) p50 / p95', [(q(tr)(0.5), 'p50'), (q(tr)(0.95), 'p95')], 's', 16)
|
||
y += 8
|
||
ts('Очередь: jobs по состоянию (PostgreSQL)', [('sum by (state) (messenger_jobs_backlog)', '{{state}}'),
|
||
('sum(messenger_outbox_pending)', 'outbox не опубликовано')], 'none', 0)
|
||
ts('Возраст старейшей job / outbox', [('max(messenger_jobs_oldest_age_seconds)', 'job (queued/processing)'),
|
||
('max(messenger_outbox_oldest_age_seconds)', 'outbox')], 's', 8,
|
||
desc='Backlog (сколько ждёт) ≠ latency обработки; alert при > 60 s')
|
||
ts('RabbitMQ: сообщения в очередях',
|
||
[('sum by (queue) (rabbitmq_detailed_queue_messages_ready or rabbitmq_queue_messages_ready)', 'ready {{queue}}'),
|
||
('sum by (queue) (rabbitmq_detailed_queue_messages_unacked or rabbitmq_queue_messages_unacked)', 'unacked {{queue}}')],
|
||
'none', 16)
|
||
y += 8
|
||
ts('Saturation worker (busy / capacity)',
|
||
[('sum(messenger_worker_busy) / sum(messenger_worker_capacity)', 'занято слотов')], 'percentunit', 0, minv=0)
|
||
ts('Publish outbox → broker (confirm) p95',
|
||
[(q('messenger_outbox_publish_duration_seconds_bucket')(0.95), 'p95'),
|
||
('sum(rate(messenger_outbox_published_total[$__rate_interval]))', 'сообщений/с')], 's', 8)
|
||
ts('Пул БД: занято / ожидания',
|
||
[('sum by (job) (pgxpool_acquired_connections)', 'занято {{job}}'),
|
||
('sum by (job) (pgxpool_max_connections)', 'max {{job}}'),
|
||
('sum by (job) (rate(pgxpool_empty_acquire_total[$__rate_interval]))', 'ожиданий/с {{job}}')], 'none', 16,
|
||
desc='empty_acquire — acquire, которому пришлось ждать свободное соединение')
|
||
y += 8
|
||
|
||
# ---------------- Ресурсы ----------------
|
||
row('Ресурсы процессов и телеметрии')
|
||
ts('CPU процессов', [('sum by (job, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))', '{{job}} {{instance}}')],
|
||
'percentunit', 0, desc='доля одного ядра; лимиты контейнеров в compose.yaml')
|
||
ts('Память Go (used)', [('sum by (job, instance) (go_memory_used_bytes)', '{{job}} {{instance}}')], 'bytes', 8)
|
||
ts('Collector: принято / отправлено / потеряно spans',
|
||
[('sum(rate({__name__=~"otelcol_receiver_accepted_spans(_total)?"}[$__rate_interval]))', 'принято'),
|
||
('sum(rate({__name__=~"otelcol_exporter_sent_spans(_total)?"}[$__rate_interval]))', 'отправлено'),
|
||
('sum(rate({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[$__rate_interval]))', 'не доставлено'),
|
||
('sum(rate({__name__=~"otelcol_receiver_refused_spans(_total)?"}[$__rate_interval]))', 'отклонено (memory_limiter)'),
|
||
('sum(otelcol_exporter_queue_size)', 'очередь экспортёров')], 'none', 16)
|
||
y += 8
|
||
|
||
# ---------------- Логи и трассы ----------------
|
||
row('Логи и трассы')
|
||
panels.append({'type': 'logs', 'title': 'Логи API и worker (ошибки и предупреждения)', 'id': nid(), 'datasource': LOKI,
|
||
'gridPos': {'h': 10, 'w': 12, 'x': 0, 'y': y},
|
||
'options': {'showTime': True, 'wrapLogMessage': True, 'enableLogDetails': True, 'sortOrder': 'Descending'},
|
||
'targets': [{'datasource': LOKI, 'refId': 'A',
|
||
'expr': '{service_name=~"messenger-(api|worker|relay)"} | detected_level=~"warn|error|WARN|ERROR"'}]})
|
||
panels.append({'type': 'table', 'title': 'Трассы worker (последние)', 'id': nid(), 'datasource': TEMPO,
|
||
'gridPos': {'h': 10, 'w': 12, 'x': 12, 'y': y},
|
||
'targets': [{'datasource': TEMPO, 'refId': 'A', 'queryType': 'traceql', 'limit': 20,
|
||
'query': '{resource.service.name="messenger-worker" && name="attachment.process process"}'}]})
|
||
y += 10
|
||
|
||
dash = {
|
||
'uid': 'messenger-overview', 'title': 'Messenger — API, worker, очередь', 'tags': ['lab6', 'messenger'],
|
||
'timezone': 'browser', 'schemaVersion': 39, 'version': 1, 'refresh': '10s',
|
||
'time': {'from': 'now-30m', 'to': 'now'}, 'editable': False, 'graphTooltip': 1,
|
||
'panels': panels, 'templating': {'list': []}, 'annotations': {'list': [
|
||
{'name': 'Alerts', 'datasource': PROM, 'enable': True, 'iconColor': 'red',
|
||
'expr': 'ALERTS{alertstate="firing"}', 'titleFormat': '{{alertname}}', 'step': '10s'}]},
|
||
}
|
||
out = pathlib.Path(__file__).resolve().parent.parent / 'infra' / 'grafana' / 'dashboards' / 'messenger.json'
|
||
out.write_text(json.dumps(dash, ensure_ascii=False, indent=1) + '\n', encoding='utf-8')
|
||
print(out, len(panels), 'panels')
|