lab6: порядок импорта в тестах, имена метрик otelcol без _total
This commit is contained in:
@@ -1355,7 +1355,7 @@
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval]))",
|
||||
"expr": "sum(rate({__name__=~\"otelcol_receiver_accepted_spans(_total)?\"}[$__rate_interval]))",
|
||||
"legendFormat": "принято",
|
||||
"refId": "A",
|
||||
"exemplar": true
|
||||
@@ -1365,7 +1365,7 @@
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum(rate(otelcol_exporter_sent_spans_total[$__rate_interval]))",
|
||||
"expr": "sum(rate({__name__=~\"otelcol_exporter_sent_spans(_total)?\"}[$__rate_interval]))",
|
||||
"legendFormat": "отправлено",
|
||||
"refId": "B",
|
||||
"exemplar": true
|
||||
@@ -1375,7 +1375,7 @@
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval]))",
|
||||
"expr": "sum(rate({__name__=~\"otelcol_exporter_send_failed_spans(_total)?\"}[$__rate_interval]))",
|
||||
"legendFormat": "не доставлено",
|
||||
"refId": "C",
|
||||
"exemplar": true
|
||||
@@ -1385,7 +1385,7 @@
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum(rate(otelcol_receiver_refused_spans_total[$__rate_interval]))",
|
||||
"expr": "sum(rate({__name__=~\"otelcol_receiver_refused_spans(_total)?\"}[$__rate_interval]))",
|
||||
"legendFormat": "отклонено (memory_limiter)",
|
||||
"refId": "D",
|
||||
"exemplar": true
|
||||
|
||||
@@ -52,7 +52,7 @@ groups:
|
||||
summary: "Нет метрик ни одного worker 1 минуту"
|
||||
runbook: "infra/RUNBOOKS.md#messengerworkerabsent"
|
||||
- alert: MessengerTelemetryExportFailing
|
||||
expr: (sum(rate(otelcol_exporter_send_failed_spans_total[1m])) or vector(0)) + (sum(rate(otelcol_exporter_send_failed_log_records_total[1m])) or vector(0)) > 0
|
||||
expr: (sum(rate({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[1m])) or vector(0)) + (sum(rate({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}[1m])) or vector(0)) > 0
|
||||
for: 30s
|
||||
labels: { severity: warning }
|
||||
annotations:
|
||||
|
||||
@@ -135,10 +135,10 @@ ts('CPU процессов', [('sum by (job, instance) (rate(process_cpu_time_se
|
||||
'percentunit', 0, desc='доля одного ядра; лимиты контейнеров в compose.yaml')
|
||||
ts('Память Go (used)', [('sum by (job, instance) (go_memory_used_bytes)', '{{job}} {{instance}}')], 'bytes', 8)
|
||||
ts('Collector: принято / отправлено / потеряно spans',
|
||||
[('sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval]))', 'принято'),
|
||||
('sum(rate(otelcol_exporter_sent_spans_total[$__rate_interval]))', 'отправлено'),
|
||||
('sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval]))', 'не доставлено'),
|
||||
('sum(rate(otelcol_receiver_refused_spans_total[$__rate_interval]))', 'отклонено (memory_limiter)'),
|
||||
[('sum(rate({__name__=~"otelcol_receiver_accepted_spans(_total)?"}[$__rate_interval]))', 'принято'),
|
||||
('sum(rate({__name__=~"otelcol_exporter_sent_spans(_total)?"}[$__rate_interval]))', 'отправлено'),
|
||||
('sum(rate({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[$__rate_interval]))', 'не доставлено'),
|
||||
('sum(rate({__name__=~"otelcol_receiver_refused_spans(_total)?"}[$__rate_interval]))', 'отклонено (memory_limiter)'),
|
||||
('sum(otelcol_exporter_queue_size)', 'очередь экспортёров')], 'none', 16)
|
||||
y += 8
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import labutil # noqa: F401 (добавляет tests/ в sys.path)
|
||||
from client import Client, Failure
|
||||
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import labutil # noqa: F401 (добавляет tests/ в sys.path)
|
||||
from client import Client, Failure, arguments, check, run
|
||||
from labutil import ROOT, compose, psql, sh, wait_status
|
||||
from loadgen import Load
|
||||
@@ -388,10 +389,10 @@ def scenario_backend_down(c):
|
||||
phase['during'] = {
|
||||
'exporter_queue_size_max': q('max_over_time(sum(otelcol_exporter_queue_size)[2m:5s])'),
|
||||
'exporter_queue_capacity': q('sum(otelcol_exporter_queue_capacity)'),
|
||||
'send_failed_spans': q('sum(increase(otelcol_exporter_send_failed_spans_total[3m]))'),
|
||||
'send_failed_log_records': q('sum(increase(otelcol_exporter_send_failed_log_records_total[3m]))'),
|
||||
'enqueue_failed_spans': q('sum(increase(otelcol_exporter_enqueue_failed_spans_total[3m]))'),
|
||||
'refused_spans_memory_limiter': q('sum(increase(otelcol_receiver_refused_spans_total[3m]))'),
|
||||
'send_failed_spans': q('sum(increase({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[3m]))'),
|
||||
'send_failed_log_records': q('sum(increase({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}[3m]))'),
|
||||
'enqueue_failed_spans': q('sum(increase({__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?"}[3m]))'),
|
||||
'refused_spans_memory_limiter': q('sum(increase({__name__=~"otelcol_receiver_refused_spans(_total)?"}[3m]))'),
|
||||
'metrics_still_scraped': q('sum(rate(http_server_request_duration_seconds_count{job="messenger-api"}[1m]))'),
|
||||
}
|
||||
compose('start', 'tempo', 'loki')
|
||||
|
||||
@@ -25,6 +25,7 @@ import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import labutil # noqa: F401 (добавляет tests/ в sys.path)
|
||||
from client import Client, arguments, check, run
|
||||
from labutil import ROOT, compose
|
||||
from loadgen import Load
|
||||
@@ -41,10 +42,10 @@ CONFIGS = {
|
||||
'COLLECTOR_CONFIG': 'collector-tail.yaml'},
|
||||
}
|
||||
COUNTERS = {
|
||||
'spans_received': 'sum(otelcol_receiver_accepted_spans_total)',
|
||||
'spans_exported': 'sum(otelcol_exporter_sent_spans_total)',
|
||||
'log_records_received': 'sum(otelcol_receiver_accepted_log_records_total)',
|
||||
'metric_points_received': 'sum(otelcol_receiver_accepted_metric_points_total)',
|
||||
'spans_received': 'sum({__name__=~"otelcol_receiver_accepted_spans(_total)?"})',
|
||||
'spans_exported': 'sum({__name__=~"otelcol_exporter_sent_spans(_total)?"})',
|
||||
'log_records_received': 'sum({__name__=~"otelcol_receiver_accepted_log_records(_total)?"})',
|
||||
'metric_points_received': 'sum({__name__=~"otelcol_receiver_accepted_metric_points(_total)?"})',
|
||||
'tempo_bytes': 'sum(tempo_distributor_bytes_received_total)',
|
||||
'loki_bytes': 'sum(loki_distributor_bytes_received_total)',
|
||||
'prometheus_samples': 'sum(prometheus_tsdb_head_samples_appended_total)',
|
||||
|
||||
Reference in New Issue
Block a user