k6 --console-output appends, so a run killed before cleanup left <run>.warm.k6 in .work and the resumed warmup counted 24 of 12. Remove the console file before each k6 run and a stale .work at start; ignore evidence/**/.work/. A telemetry storage that exits during the lookup (Tempo in metrics-r3: name no longer resolvable) now fails the run with the container state, OOMKilled flag and its log tail saved to raw/; Tempo requests of the lookup are limited to 3 at a time and memory peaks after the lookup are recorded. Raise the light classes of the telemetry cost load by 15 % (read 126.5, write 23, me 34.5, card 34.5 per second): on a quiet host `full` kept the replicas at 28-29.5 % CPU, below the 30 % calibration bound. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
641 lines
24 KiB
Python
641 lines
24 KiB
Python
"""Выборка из хранилищ телеметрии профиля цены телеметрии.
|
||
|
||
Design change `track1-telemetry-cost`, решение 9. Только в контейнере
|
||
`mtusi-bench-lookup` на образе стенда, в сети хранилищ `mtusi-bench-telemetry`;
|
||
только stdlib (`urllib`), `src` не импортирует, `bench/` — на чтение. Запросы
|
||
— HTTP API Tempo, Loki, Prometheus и метрики Collector, пулом из восьми
|
||
потоков; к Tempo — не больше трёх запросов одновременно.
|
||
|
||
python telemetry_lookup.py wait-ready [--timeout 90]
|
||
python telemetry_lookup.py collect <события.json> <итог.json>
|
||
|
||
`wait-ready` — `/-/ready` Prometheus, `/ready` Loki и Tempo, `:8888/metrics`
|
||
Collector; код 0 — все готовы, 1 — срок вышел. `collect` — по событиям прогона
|
||
(id запросов редких классов, выборка нормальных, окна) ищет трассы по
|
||
`span.messenger.request_id` (и цельные цепочки загрузок), записи журнала по
|
||
`request_id` и их переход к трассе, счётчики Prometheus (приём, потери,
|
||
события), ряды по `job`, `status/tsdb`, признаки слоёв Collector и
|
||
Prometheus, квантили из buckets против журнала. Результат поиска, равный
|
||
пределу числа результатов, — отказ «lookup truncated» (код 1): неполный счёт
|
||
не выдаётся за полный.
|
||
"""
|
||
|
||
import contextlib
|
||
import json
|
||
import math
|
||
import sys
|
||
import threading
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from typing import Any
|
||
|
||
PROMETHEUS = "http://prometheus:9090"
|
||
LOKI = "http://loki:3100"
|
||
TEMPO = "http://tempo:3200"
|
||
COLLECTOR = "http://otel-collector:8888"
|
||
READY = (
|
||
f"{PROMETHEUS}/-/ready",
|
||
f"{LOKI}/ready",
|
||
f"{TEMPO}/ready",
|
||
f"{COLLECTOR}/metrics",
|
||
)
|
||
THREADS = 8
|
||
# переходы журнал → трасса: потоков на класс (классы идут параллельно)
|
||
LINK_THREADS = 2
|
||
# одновременных запросов к Tempo (поиски классов и переходы журнал → трасса):
|
||
# под восемью поисками и загрузками трасс сразу Tempo с лимитом рабочего
|
||
# стенда (640m без swap) завершился посреди выборки metrics-r3 метки baseline
|
||
# (design, «Уточнения apply», п. 16); выборка идёт после окон замера
|
||
TEMPO_REQUESTS = 3
|
||
TEMPO_GATE = threading.BoundedSemaphore(TEMPO_REQUESTS)
|
||
TIMEOUT_S = 60
|
||
ATTEMPTS = 4
|
||
RETRY_PAUSE_S = 1.0
|
||
# счётчики по краям окна: шаг — интервал scrape, взгляд назад — был ли ряд
|
||
# до окна
|
||
STEP_S = 15
|
||
LOOKBACK_S = 60
|
||
# id в одном поиске Tempo и Loki: регулярное выражение в query string
|
||
BATCH = 40
|
||
# предел результатов — не меньше 4 × ожидаемого (решение 9)
|
||
LIMIT_FACTOR = 4
|
||
API_SERVICE = "messenger-api"
|
||
REQUEST_ATTR = "span.messenger.request_id"
|
||
# у загрузки цепочка — server span API, span producer relay, consumer span
|
||
# worker с итогом задания
|
||
CHAIN_OUTCOME = {"upload": "ready", "upload_bad": "failed"}
|
||
# имена Collector — без суффикса _total у собственных метрик (rule
|
||
# telemetry.md); на всякий случай оба варианта
|
||
TOTALS = {
|
||
"sdk_span_started_sampled": (
|
||
'sum(otel_sdk_span_started_total{otel_span_sampling_result='
|
||
'"RECORD_AND_SAMPLE"})'
|
||
),
|
||
"sdk_span_started_dropped": (
|
||
'sum(otel_sdk_span_started_total{otel_span_sampling_result="DROP"})'
|
||
),
|
||
"sdk_span_processed_ok": (
|
||
'sum(otel_sdk_processor_span_processed_total{error_type=""})'
|
||
),
|
||
"sdk_span_processed_lost": (
|
||
'sum(otel_sdk_processor_span_processed_total{error_type!=""})'
|
||
),
|
||
"sdk_span_exported_ok": (
|
||
'sum(otel_sdk_exporter_span_exported_total{error_type=""})'
|
||
),
|
||
"sdk_span_export_failed": (
|
||
'sum(otel_sdk_exporter_span_exported_total{error_type!=""})'
|
||
),
|
||
"sdk_log_processed_ok": (
|
||
'sum(otel_sdk_processor_log_processed_total{error_type=""})'
|
||
),
|
||
"sdk_log_processed_lost": (
|
||
'sum(otel_sdk_processor_log_processed_total{error_type!=""})'
|
||
),
|
||
"sdk_log_exported_ok": (
|
||
'sum(otel_sdk_exporter_log_exported_total{error_type=""})'
|
||
),
|
||
"sdk_log_export_failed": (
|
||
'sum(otel_sdk_exporter_log_exported_total{error_type!=""})'
|
||
),
|
||
"col_accepted_spans": "otelcol_receiver_accepted_spans",
|
||
"col_refused_spans": "otelcol_receiver_refused_spans",
|
||
"col_failed_spans": "otelcol_receiver_failed_spans",
|
||
"col_accepted_logs": "otelcol_receiver_accepted_log_records",
|
||
"col_refused_logs": "otelcol_receiver_refused_log_records",
|
||
"col_failed_logs": "otelcol_receiver_failed_log_records",
|
||
"col_accepted_points": "otelcol_receiver_accepted_metric_points",
|
||
"col_refused_points": "otelcol_receiver_refused_metric_points",
|
||
"col_sent_spans": "otelcol_exporter_sent_spans",
|
||
"col_send_failed_spans": "otelcol_exporter_send_failed_spans",
|
||
"col_enqueue_failed_spans": "otelcol_exporter_enqueue_failed_spans",
|
||
"col_sent_logs": "otelcol_exporter_sent_log_records",
|
||
"col_send_failed_logs": "otelcol_exporter_send_failed_log_records",
|
||
"col_enqueue_failed_logs": "otelcol_exporter_enqueue_failed_log_records",
|
||
"col_sent_points": "otelcol_exporter_sent_metric_points",
|
||
"tail_sampled": (
|
||
"otelcol_processor_tail_sampling_global_count_traces_sampled"
|
||
'{sampled="true"}'
|
||
),
|
||
"tail_not_sampled": (
|
||
"otelcol_processor_tail_sampling_global_count_traces_sampled"
|
||
'{sampled="false"}'
|
||
),
|
||
"tail_dropped_too_early": (
|
||
"otelcol_processor_tail_sampling_sampling_trace_dropped_too_early"
|
||
),
|
||
"tail_new_traces": (
|
||
"otelcol_processor_tail_sampling_new_trace_id_received"
|
||
),
|
||
"tail_policy_errors": (
|
||
"otelcol_processor_tail_sampling_sampling_policy_evaluation_error"
|
||
),
|
||
"tempo_spans": "sum(tempo_distributor_spans_received_total)",
|
||
"tempo_bytes": "sum(tempo_distributor_bytes_received_total)",
|
||
"loki_lines": "sum(loki_distributor_lines_received_total)",
|
||
"loki_bytes": "sum(loki_distributor_bytes_received_total)",
|
||
"loki_sm_bytes": (
|
||
"sum(loki_distributor_structured_metadata_bytes_received_total)"
|
||
),
|
||
"prom_series": "max(prometheus_tsdb_head_series)",
|
||
"prom_samples": "sum(prometheus_tsdb_head_samples_appended_total)",
|
||
}
|
||
# счётчики, приращение которых нужно за чистое окно (приём, решение 9)
|
||
CLEAN = (
|
||
"col_accepted_spans",
|
||
"col_accepted_logs",
|
||
"col_accepted_points",
|
||
"tempo_spans",
|
||
"tempo_bytes",
|
||
"loki_lines",
|
||
"loki_bytes",
|
||
"loki_sm_bytes",
|
||
"prom_samples",
|
||
"prom_series",
|
||
)
|
||
GAUGES = frozenset({"prom_series"})
|
||
CONTENT_ROUTE = "/api/v1/attachments/{attachment_id}/content"
|
||
MESSAGES_ROUTE = "/api/v1/chats/{chat_id}/messages"
|
||
HTTP_COUNT = "http_server_request_duration_seconds_count"
|
||
HTTP_BUCKET = "http_server_request_duration_seconds_bucket"
|
||
RARE_METRICS = {
|
||
"broken": (
|
||
f'sum({HTTP_COUNT}{{http_route="{CONTENT_ROUTE}",'
|
||
'http_response_status_code="500"})'
|
||
),
|
||
"register": (
|
||
f'sum({HTTP_COUNT}{{http_route="/api/v1/users",'
|
||
'http_request_method="POST",http_response_status_code="201"})'
|
||
),
|
||
"write_slow": (
|
||
f'sum({HTTP_COUNT}{{http_route="{MESSAGES_ROUTE}",'
|
||
'http_request_method="POST"}) - '
|
||
f'sum({HTTP_BUCKET}{{http_route="{MESSAGES_ROUTE}",'
|
||
'http_request_method="POST",le="0.25"})'
|
||
),
|
||
"upload_bad": (
|
||
'sum(messenger_job_duration_seconds_count{messenger_job_outcome='
|
||
'"failed"})'
|
||
),
|
||
"upload": "sum(messenger_job_time_to_ready_seconds_count)",
|
||
}
|
||
QUANTILES = (0.95, 0.99)
|
||
|
||
|
||
class TruncatedError(Exception):
|
||
"""Поиск упёрся в предел числа результатов."""
|
||
|
||
|
||
def collector_name(expr: str) -> str:
|
||
"""Имя Collector с и без `_total`: `sum({__name__=~"имя(_total)?", …})`."""
|
||
if expr.startswith(("sum(", "max(")):
|
||
return expr
|
||
name, _, labels = expr.partition("{")
|
||
labels = labels.rstrip("}")
|
||
matcher = f'__name__=~"{name}(_total)?"'
|
||
if labels:
|
||
matcher += f",{labels}"
|
||
return f"sum({{{matcher}}})"
|
||
|
||
|
||
def fetch(url: str) -> bytes:
|
||
"""GET с повтором на обрыв соединения и 5xx: хранилище с 0,5 CPU под
|
||
пулом запросов изредка закрывает соединение без ответа. 4xx — сразу.
|
||
Запрос к Tempo ждёт места `TEMPO_GATE`; пауза повтора места не держит."""
|
||
for attempt in range(1, ATTEMPTS + 1):
|
||
gate = (
|
||
TEMPO_GATE if url.startswith(TEMPO) else contextlib.nullcontext()
|
||
)
|
||
try:
|
||
with (
|
||
gate,
|
||
urllib.request.urlopen(url, timeout=TIMEOUT_S) as response,
|
||
):
|
||
return response.read()
|
||
except urllib.error.HTTPError as exc:
|
||
if exc.code < 500 or attempt == ATTEMPTS:
|
||
raise
|
||
except (OSError, urllib.error.URLError):
|
||
if attempt == ATTEMPTS:
|
||
raise
|
||
time.sleep(RETRY_PAUSE_S * attempt)
|
||
raise AssertionError("unreachable")
|
||
|
||
|
||
def get(url: str, params: dict[str, Any] | None = None) -> Any:
|
||
if params:
|
||
url = f"{url}?{urllib.parse.urlencode(params)}"
|
||
body = fetch(url)
|
||
return json.loads(body) if body[:1] in (b"{", b"[") else body
|
||
|
||
|
||
def exists(url: str) -> bool:
|
||
try:
|
||
fetch(url)
|
||
except urllib.error.HTTPError as exc:
|
||
if exc.code == 404:
|
||
return False
|
||
raise
|
||
return True
|
||
|
||
|
||
def wait_ready(timeout: float) -> int:
|
||
deadline = time.monotonic() + timeout
|
||
pending = list(READY)
|
||
while time.monotonic() < deadline:
|
||
left = []
|
||
for url in pending:
|
||
try:
|
||
with urllib.request.urlopen(url, timeout=3) as response:
|
||
response.read()
|
||
if response.status != 200:
|
||
left.append(url)
|
||
except (OSError, urllib.error.URLError):
|
||
left.append(url)
|
||
pending = left
|
||
if not pending:
|
||
print("telemetry storages ready", flush=True)
|
||
return 0
|
||
time.sleep(1)
|
||
print(f"not ready in {timeout:.0f}s: {pending}", flush=True)
|
||
return 1
|
||
|
||
|
||
def batches(items: list[str]) -> list[list[str]]:
|
||
return [items[i : i + BATCH] for i in range(0, len(items), BATCH)]
|
||
|
||
|
||
def tempo_search(query: str, window: dict[str, float], expected: int) -> Any:
|
||
limit = max(20, LIMIT_FACTOR * expected)
|
||
data = get(
|
||
f"{TEMPO}/api/search",
|
||
{
|
||
"q": query,
|
||
"start": int(window["start"]),
|
||
"end": int(window["end"]) + 1,
|
||
"limit": limit,
|
||
"spss": 10,
|
||
},
|
||
)
|
||
traces = data.get("traces") or []
|
||
if len(traces) >= limit:
|
||
raise TruncatedError(
|
||
f"lookup truncated: tempo {len(traces)} = {limit}",
|
||
)
|
||
return traces
|
||
|
||
|
||
def request_ids_in(traces: list[dict[str, Any]]) -> set[str]:
|
||
found = set()
|
||
for item in traces:
|
||
for spanset in item.get("spanSets") or [item.get("spanSet") or {}]:
|
||
for span in spanset.get("spans") or []:
|
||
for attr in span.get("attributes") or []:
|
||
if attr.get("key") == "messenger.request_id":
|
||
found.add(attr["value"].get("stringValue", ""))
|
||
return found
|
||
|
||
|
||
def traces_of(
|
||
ids: list[str],
|
||
window: dict[str, float],
|
||
outcome: str | None = None,
|
||
) -> set[str]:
|
||
"""Id запросов, у которых в Tempo есть трасса (у `outcome` — цельная
|
||
цепочка загрузки с consumer span этого итога и span producer)."""
|
||
found: set[str] = set()
|
||
for part in batches(ids):
|
||
pattern = "|".join(part)
|
||
query = f'{{ {REQUEST_ATTR} =~ "{pattern}" }}'
|
||
if outcome is not None:
|
||
query += (
|
||
" && { kind = consumer && span.messenger.job.outcome = "
|
||
f'"{outcome}" }} && {{ kind = producer }}'
|
||
)
|
||
query += f" | select({REQUEST_ATTR})"
|
||
found |= request_ids_in(tempo_search(query, window, len(part)))
|
||
return found & set(ids)
|
||
|
||
|
||
def loki_records(
|
||
ids: list[str],
|
||
window: dict[str, float],
|
||
) -> dict[str, dict[str, str]]:
|
||
"""Запись доступа API по `request_id`: метки потока (поля записи)."""
|
||
out: dict[str, dict[str, str]] = {}
|
||
for part in batches(ids):
|
||
limit = LIMIT_FACTOR * len(part) + 10
|
||
query = (
|
||
f'{{service_name="{API_SERVICE}"}} | middleware="http" | '
|
||
f'request_id=~"{"|".join(part)}"'
|
||
)
|
||
data = get(
|
||
f"{LOKI}/loki/api/v1/query_range",
|
||
{
|
||
"query": query,
|
||
"start": int(window["start"] * 1e9),
|
||
"end": int((window["end"] + 1) * 1e9),
|
||
"limit": limit,
|
||
"direction": "forward",
|
||
},
|
||
)
|
||
streams = data["data"]["result"]
|
||
lines = sum(len(stream["values"]) for stream in streams)
|
||
if lines >= limit:
|
||
raise TruncatedError(f"lookup truncated: loki {lines} = {limit}")
|
||
for stream in streams:
|
||
labels = stream["stream"]
|
||
out[labels.get("request_id", "")] = labels
|
||
return {key: value for key, value in out.items() if key in set(ids)}
|
||
|
||
|
||
def trace_exists(trace_id: str) -> bool:
|
||
"""Трасса есть и в ней есть span: на неизвестный id Tempo может ответить
|
||
и пустым телом (пробный прогон `head` — 12 «найденных» трасс из 12
|
||
невыбранных)."""
|
||
try:
|
||
body = fetch(f"{TEMPO}/api/v2/traces/{trace_id}")
|
||
except urllib.error.HTTPError as exc:
|
||
if exc.code == 404:
|
||
return False
|
||
raise
|
||
if not body.strip():
|
||
return False
|
||
data = json.loads(body)
|
||
trace = data.get("trace", data)
|
||
return bool(trace.get("resourceSpans") or trace.get("batches"))
|
||
|
||
|
||
def class_result(
|
||
name: str,
|
||
ids: list[str],
|
||
window: dict[str, float],
|
||
slow_ms: float,
|
||
) -> dict[str, Any]:
|
||
traced = traces_of(ids, window)
|
||
chains = None
|
||
if name in CHAIN_OUTCOME and ids:
|
||
chains = len(traces_of(ids, window, CHAIN_OUTCOME[name]))
|
||
records = loki_records(ids, window) if ids else {}
|
||
with_trace = {
|
||
rid: labels["trace_id"]
|
||
for rid, labels in records.items()
|
||
if labels.get("trace_id")
|
||
}
|
||
sampled = sum(
|
||
1
|
||
for labels in records.values()
|
||
if labels.get("flags", "").isdigit() and int(labels["flags"]) & 1
|
||
)
|
||
# свой пул: задачи классов уже заняли потоки общего
|
||
with ThreadPoolExecutor(LINK_THREADS) as links:
|
||
linked_by = dict(
|
||
zip(
|
||
with_trace,
|
||
links.map(trace_exists, with_trace.values()),
|
||
strict=True,
|
||
),
|
||
)
|
||
linked = sum(linked_by.values())
|
||
# медленные по серверу: длительность записи доступа (та же величина, что
|
||
# server span) не меньше порога; по ней решает правило выбора трасс
|
||
server_slow = {
|
||
rid
|
||
for rid, labels in records.items()
|
||
if float(labels.get("duration_ms") or 0) >= slow_ms
|
||
}
|
||
return {
|
||
"ids": len(ids),
|
||
"traces": len(traced),
|
||
"server_slow": len(server_slow),
|
||
"server_slow_traces": len(server_slow & traced),
|
||
"server_slow_linked": sum(
|
||
1 for rid in server_slow if linked_by.get(rid)
|
||
),
|
||
"chains": chains,
|
||
"logs": len(records),
|
||
"logs_with_trace": len(with_trace),
|
||
"logs_sampled_flag": sampled,
|
||
"log_to_trace": linked,
|
||
"missing_traces": sorted(set(ids) - traced)[:5],
|
||
}
|
||
|
||
|
||
def prom_value(expr: str, at: float) -> float | None:
|
||
data = get(f"{PROMETHEUS}/api/v1/query", {"query": expr, "time": at})
|
||
result = data["data"]["result"]
|
||
if not result:
|
||
return None
|
||
value = float(result[0]["value"][1])
|
||
# histogram_quantile без отсчётов — NaN: в JSON это не число
|
||
return None if math.isnan(value) else value
|
||
|
||
|
||
def prom_delta(
|
||
expr: str,
|
||
start: float,
|
||
end: float,
|
||
*,
|
||
gauge: bool,
|
||
) -> dict[str, float | None]:
|
||
"""Разность первого отсчёта не раньше начала окна и последнего не позже
|
||
конца (шаг 15 с, решение 9); у gauge — последнее значение."""
|
||
data = get(
|
||
f"{PROMETHEUS}/api/v1/query_range",
|
||
{
|
||
"query": expr,
|
||
"start": start - LOOKBACK_S,
|
||
"end": end,
|
||
"step": STEP_S,
|
||
},
|
||
)
|
||
result = data["data"]["result"]
|
||
if not result:
|
||
return {"delta": None, "seconds": None, "last": None}
|
||
series = [(float(t), float(v)) for t, v in result[0]["values"]]
|
||
inside = [(t, v) for t, v in series if start <= t <= end]
|
||
if not inside:
|
||
return {"delta": None, "seconds": None, "last": None}
|
||
t1, v1 = inside[-1]
|
||
if gauge:
|
||
return {"delta": None, "seconds": t1 - inside[0][0], "last": v1}
|
||
# ряд счётчика, которого до окна не было (Tempo заводит его на первый
|
||
# принятый span — у tail через срок решения), начинается с нуля: база —
|
||
# ноль в начале окна, а не первый отсчёт внутри (иначе приёма «нет»)
|
||
if any(t < start for t, _ in series) and len(inside) >= 2:
|
||
t0, v0 = inside[0]
|
||
else:
|
||
t0, v0 = start, 0.0
|
||
if t1 <= t0:
|
||
return {"delta": None, "seconds": None, "last": v1}
|
||
return {"delta": v1 - v0, "seconds": t1 - t0, "last": v1}
|
||
|
||
|
||
def quantiles(
|
||
method: str,
|
||
route: str,
|
||
window: dict[str, float],
|
||
) -> dict[str, float | None]:
|
||
seconds = max(1, int(window["end"] - window["start"]))
|
||
out: dict[str, float | None] = {}
|
||
for q in QUANTILES:
|
||
key = f"p{int(q * 100)}"
|
||
out[f"{key}_buckets"] = prom_value(
|
||
f"histogram_quantile({q}, sum by (le) (increase({HTTP_BUCKET}"
|
||
f'{{http_route="{route}",http_request_method="{method}"}}'
|
||
f"[{seconds}s])))",
|
||
window["end"],
|
||
)
|
||
data = get(
|
||
f"{LOKI}/loki/api/v1/query",
|
||
{
|
||
"query": (
|
||
f"quantile_over_time({q}, "
|
||
f'{{service_name="{API_SERVICE}"}} | middleware="http" '
|
||
f'| method="{method}" | path="{route}" '
|
||
f"| unwrap duration_ms [{seconds}s]) by ()"
|
||
),
|
||
"time": int(window["end"] * 1e9),
|
||
},
|
||
)
|
||
result = data["data"]["result"]
|
||
value = float(result[0]["value"][1]) / 1000 if result else None
|
||
out[f"{key}_log"] = (
|
||
None if value is None or math.isnan(value) else value
|
||
)
|
||
return out
|
||
|
||
|
||
def loki_counts(window: dict[str, float]) -> dict[str, int]:
|
||
seconds = max(1, int(window["end"] - window["start"]))
|
||
data = get(
|
||
f"{LOKI}/loki/api/v1/query",
|
||
{
|
||
"query": (
|
||
"sum by (method, path) (count_over_time("
|
||
f'{{service_name="{API_SERVICE}"}} | middleware="http" '
|
||
f"[{seconds}s]))"
|
||
),
|
||
"time": int(window["end"] * 1e9),
|
||
},
|
||
)
|
||
return {
|
||
f"{item['metric'].get('method', '')} {item['metric'].get('path', '')}": (
|
||
int(float(item["value"][1]))
|
||
)
|
||
for item in data["data"]["result"]
|
||
}
|
||
|
||
|
||
def collect(events: dict[str, Any]) -> dict[str, Any]:
|
||
started = time.monotonic()
|
||
search = events["search"]
|
||
run = events["run"]
|
||
clean = events["clean"]
|
||
out: dict[str, Any] = {"classes": {}}
|
||
with ThreadPoolExecutor(THREADS) as pool:
|
||
futures = {
|
||
name: pool.submit(
|
||
class_result,
|
||
name,
|
||
ids,
|
||
search,
|
||
float(events.get("slow_ms", 250)),
|
||
)
|
||
for name, ids in events["classes"].items()
|
||
}
|
||
sample = events.get("sample_me") or []
|
||
sample_future = pool.submit(traces_of, sample, search)
|
||
for name, future in futures.items():
|
||
out["classes"][name] = future.result()
|
||
out["sample_me"] = {
|
||
"ids": len(sample),
|
||
"traces": len(sample_future.result()),
|
||
}
|
||
at = search["end"]
|
||
names = list(TOTALS)
|
||
values = pool.map(
|
||
lambda key: prom_value(collector_name(TOTALS[key]), at),
|
||
names,
|
||
)
|
||
out["totals"] = dict(zip(names, values, strict=True))
|
||
rare = list(RARE_METRICS)
|
||
out["rare_metrics"] = dict(
|
||
zip(
|
||
rare,
|
||
pool.map(lambda key: prom_value(RARE_METRICS[key], at), rare),
|
||
strict=True,
|
||
),
|
||
)
|
||
out["clean"] = dict(
|
||
zip(
|
||
CLEAN,
|
||
pool.map(
|
||
lambda key: prom_delta(
|
||
collector_name(TOTALS[key]),
|
||
clean["start"],
|
||
clean["end"],
|
||
gauge=key in GAUGES,
|
||
),
|
||
CLEAN,
|
||
),
|
||
strict=True,
|
||
),
|
||
)
|
||
out["quantiles"] = {
|
||
name: quantiles(method, route, clean)
|
||
for name, (method, route) in events["routes"].items()
|
||
}
|
||
out["loki_counts"] = loki_counts(run)
|
||
jobs = get(
|
||
f"{PROMETHEUS}/api/v1/query",
|
||
{"query": 'count by (job) ({__name__=~".+"})', "time": at},
|
||
)
|
||
out["series_by_job"] = {
|
||
item["metric"].get("job", ""): int(float(item["value"][1]))
|
||
for item in jobs["data"]["result"]
|
||
}
|
||
tsdb = get(f"{PROMETHEUS}/api/v1/status/tsdb")["data"]
|
||
out["tsdb"] = {
|
||
"num_series": tsdb.get("headStats", {}).get("numSeries"),
|
||
"top_metrics": tsdb.get("seriesCountByMetricName", [])[:10],
|
||
}
|
||
config = get(f"{PROMETHEUS}/api/v1/status/config")["data"]["yaml"]
|
||
tail = prom_value(
|
||
'count({__name__=~"otelcol_processor_tail_sampling_.*"})',
|
||
at,
|
||
)
|
||
out["layers"] = {
|
||
"tail_sampling": bool(tail),
|
||
"lean_prometheus": "loki_distributor_" in config,
|
||
}
|
||
out["seconds"] = round(time.monotonic() - started, 1)
|
||
return out
|
||
|
||
|
||
def main(argv: list[str]) -> int:
|
||
if argv[:1] == ["wait-ready"]:
|
||
timeout = float(argv[2]) if argv[1:2] == ["--timeout"] else 90.0
|
||
return wait_ready(timeout)
|
||
if argv[:1] == ["collect"] and len(argv) == 3:
|
||
with open(argv[1], encoding="utf-8") as file:
|
||
events = json.load(file)
|
||
try:
|
||
result = collect(events)
|
||
except TruncatedError as exc:
|
||
print(str(exc), flush=True)
|
||
return 1
|
||
with open(argv[2], "w", encoding="utf-8") as file:
|
||
json.dump(result, file, ensure_ascii=False, indent=1)
|
||
print(f"lookup done in {result['seconds']}s", flush=True)
|
||
return 0
|
||
print(__doc__, flush=True)
|
||
return 2
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main(sys.argv[1:]))
|