From 2c09b347f213ecf634f9a27edee4d8a33ea728e3 Mon Sep 17 00:00:00 2001 From: CharaWein Date: Thu, 12 Nov 2026 12:27:18 +0300 Subject: [PATCH] =?UTF-8?q?lab6:=20=D0=BF=D0=BE=D1=80=D1=8F=D0=B4=D0=BE?= =?UTF-8?q?=D0=BA=20=D0=B8=D0=BC=D0=BF=D0=BE=D1=80=D1=82=D0=B0=20=D0=B2=20?= =?UTF-8?q?=D1=82=D0=B5=D1=81=D1=82=D0=B0=D1=85,=20=D0=B8=D0=BC=D0=B5?= =?UTF-8?q?=D0=BD=D0=B0=20=D0=BC=D0=B5=D1=82=D1=80=D0=B8=D0=BA=20otelcol?= =?UTF-8?q?=20=D0=B1=D0=B5=D0=B7=20=5Ftotal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- infra/grafana/dashboards/messenger.json | 8 ++++---- infra/prometheus/rules/messenger.yml | 2 +- scripts/gen_dashboard.py | 8 ++++---- student-tests/loadgen.py | 1 + student-tests/observability_ops.py | 9 +++++---- student-tests/telemetry_cost.py | 9 +++++---- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json index 1a52283..d444351 100644 --- a/infra/grafana/dashboards/messenger.json +++ b/infra/grafana/dashboards/messenger.json @@ -1355,7 +1355,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval]))", + "expr": "sum(rate({__name__=~\"otelcol_receiver_accepted_spans(_total)?\"}[$__rate_interval]))", "legendFormat": "принято", "refId": "A", "exemplar": true @@ -1365,7 +1365,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(otelcol_exporter_sent_spans_total[$__rate_interval]))", + "expr": "sum(rate({__name__=~\"otelcol_exporter_sent_spans(_total)?\"}[$__rate_interval]))", "legendFormat": "отправлено", "refId": "B", "exemplar": true @@ -1375,7 +1375,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval]))", + "expr": "sum(rate({__name__=~\"otelcol_exporter_send_failed_spans(_total)?\"}[$__rate_interval]))", "legendFormat": "не доставлено", "refId": "C", "exemplar": true @@ -1385,7 +1385,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(otelcol_receiver_refused_spans_total[$__rate_interval]))", + "expr": "sum(rate({__name__=~\"otelcol_receiver_refused_spans(_total)?\"}[$__rate_interval]))", "legendFormat": "отклонено (memory_limiter)", "refId": "D", "exemplar": true diff --git a/infra/prometheus/rules/messenger.yml b/infra/prometheus/rules/messenger.yml index 108499b..ea32cee 100644 --- a/infra/prometheus/rules/messenger.yml +++ b/infra/prometheus/rules/messenger.yml @@ -52,7 +52,7 @@ groups: summary: "Нет метрик ни одного worker 1 минуту" runbook: "infra/RUNBOOKS.md#messengerworkerabsent" - alert: MessengerTelemetryExportFailing - expr: (sum(rate(otelcol_exporter_send_failed_spans_total[1m])) or vector(0)) + (sum(rate(otelcol_exporter_send_failed_log_records_total[1m])) or vector(0)) > 0 + expr: (sum(rate({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[1m])) or vector(0)) + (sum(rate({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}[1m])) or vector(0)) > 0 for: 30s labels: { severity: warning } annotations: diff --git a/scripts/gen_dashboard.py b/scripts/gen_dashboard.py index d58aabb..1ae85f3 100644 --- a/scripts/gen_dashboard.py +++ b/scripts/gen_dashboard.py @@ -135,10 +135,10 @@ ts('CPU процессов', [('sum by (job, instance) (rate(process_cpu_time_se 'percentunit', 0, desc='доля одного ядра; лимиты контейнеров в compose.yaml') ts('Память Go (used)', [('sum by (job, instance) (go_memory_used_bytes)', '{{job}} {{instance}}')], 'bytes', 8) ts('Collector: принято / отправлено / потеряно spans', - [('sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval]))', 'принято'), - ('sum(rate(otelcol_exporter_sent_spans_total[$__rate_interval]))', 'отправлено'), - ('sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval]))', 'не доставлено'), - ('sum(rate(otelcol_receiver_refused_spans_total[$__rate_interval]))', 'отклонено (memory_limiter)'), + [('sum(rate({__name__=~"otelcol_receiver_accepted_spans(_total)?"}[$__rate_interval]))', 'принято'), + ('sum(rate({__name__=~"otelcol_exporter_sent_spans(_total)?"}[$__rate_interval]))', 'отправлено'), + ('sum(rate({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[$__rate_interval]))', 'не доставлено'), + ('sum(rate({__name__=~"otelcol_receiver_refused_spans(_total)?"}[$__rate_interval]))', 'отклонено (memory_limiter)'), ('sum(otelcol_exporter_queue_size)', 'очередь экспортёров')], 'none', 16) y += 8 diff --git a/student-tests/loadgen.py b/student-tests/loadgen.py index 4ee6b17..8834a35 100644 --- a/student-tests/loadgen.py +++ b/student-tests/loadgen.py @@ -10,6 +10,7 @@ import time import uuid from concurrent.futures import ThreadPoolExecutor +import labutil # noqa: F401 (добавляет tests/ в sys.path) from client import Client, Failure diff --git a/student-tests/observability_ops.py b/student-tests/observability_ops.py index 660bacb..bcab5d9 100644 --- a/student-tests/observability_ops.py +++ b/student-tests/observability_ops.py @@ -25,6 +25,7 @@ import time import uuid from pathlib import Path +import labutil # noqa: F401 (добавляет tests/ в sys.path) from client import Client, Failure, arguments, check, run from labutil import ROOT, compose, psql, sh, wait_status from loadgen import Load @@ -388,10 +389,10 @@ def scenario_backend_down(c): phase['during'] = { 'exporter_queue_size_max': q('max_over_time(sum(otelcol_exporter_queue_size)[2m:5s])'), 'exporter_queue_capacity': q('sum(otelcol_exporter_queue_capacity)'), - 'send_failed_spans': q('sum(increase(otelcol_exporter_send_failed_spans_total[3m]))'), - 'send_failed_log_records': q('sum(increase(otelcol_exporter_send_failed_log_records_total[3m]))'), - 'enqueue_failed_spans': q('sum(increase(otelcol_exporter_enqueue_failed_spans_total[3m]))'), - 'refused_spans_memory_limiter': q('sum(increase(otelcol_receiver_refused_spans_total[3m]))'), + 'send_failed_spans': q('sum(increase({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[3m]))'), + 'send_failed_log_records': q('sum(increase({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}[3m]))'), + 'enqueue_failed_spans': q('sum(increase({__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?"}[3m]))'), + 'refused_spans_memory_limiter': q('sum(increase({__name__=~"otelcol_receiver_refused_spans(_total)?"}[3m]))'), 'metrics_still_scraped': q('sum(rate(http_server_request_duration_seconds_count{job="messenger-api"}[1m]))'), } compose('start', 'tempo', 'loki') diff --git a/student-tests/telemetry_cost.py b/student-tests/telemetry_cost.py index 6d12780..f049446 100644 --- a/student-tests/telemetry_cost.py +++ b/student-tests/telemetry_cost.py @@ -25,6 +25,7 @@ import time import uuid from pathlib import Path +import labutil # noqa: F401 (добавляет tests/ в sys.path) from client import Client, arguments, check, run from labutil import ROOT, compose from loadgen import Load @@ -41,10 +42,10 @@ CONFIGS = { 'COLLECTOR_CONFIG': 'collector-tail.yaml'}, } COUNTERS = { - 'spans_received': 'sum(otelcol_receiver_accepted_spans_total)', - 'spans_exported': 'sum(otelcol_exporter_sent_spans_total)', - 'log_records_received': 'sum(otelcol_receiver_accepted_log_records_total)', - 'metric_points_received': 'sum(otelcol_receiver_accepted_metric_points_total)', + 'spans_received': 'sum({__name__=~"otelcol_receiver_accepted_spans(_total)?"})', + 'spans_exported': 'sum({__name__=~"otelcol_exporter_sent_spans(_total)?"})', + 'log_records_received': 'sum({__name__=~"otelcol_receiver_accepted_log_records(_total)?"})', + 'metric_points_received': 'sum({__name__=~"otelcol_receiver_accepted_metric_points(_total)?"})', 'tempo_bytes': 'sum(tempo_distributor_bytes_received_total)', 'loki_bytes': 'sum(loki_distributor_bytes_received_total)', 'prometheus_samples': 'sum(prometheus_tsdb_head_samples_appended_total)',