Files
mtusi2026lab6/bench/telemetry_lookup.py
T
ricoromaandClaude Opus 5.5 68e14b6ddf fix(bench): clear stale k6 console files and check telemetry storages around lookup
k6 --console-output appends, so a run killed before cleanup left
<run>.warm.k6 in .work and the resumed warmup counted 24 of 12.
Remove the console file before each k6 run and a stale .work at
start; ignore evidence/**/.work/.

A telemetry storage that exits during the lookup (Tempo in
metrics-r3: name no longer resolvable) now fails the run with the
container state, OOMKilled flag and its log tail saved to raw/;
Tempo requests of the lookup are limited to 3 at a time and
memory peaks after the lookup are recorded.

Raise the light classes of the telemetry cost load by 15 % (read
126.5, write 23, me 34.5, card 34.5 per second): on a quiet host
`full` kept the replicas at 28-29.5 % CPU, below the 30 %
calibration bound.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 23:42:53 +03:00

641 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Выборка из хранилищ телеметрии профиля цены телеметрии.
Design change `track1-telemetry-cost`, решение 9. Только в контейнере
`mtusi-bench-lookup` на образе стенда, в сети хранилищ `mtusi-bench-telemetry`;
только stdlib (`urllib`), `src` не импортирует, `bench/` — на чтение. Запросы
— HTTP API Tempo, Loki, Prometheus и метрики Collector, пулом из восьми
потоков; к Tempo — не больше трёх запросов одновременно.
python telemetry_lookup.py wait-ready [--timeout 90]
python telemetry_lookup.py collect <события.json> <итог.json>
`wait-ready` — `/-/ready` Prometheus, `/ready` Loki и Tempo, `:8888/metrics`
Collector; код 0 — все готовы, 1 — срок вышел. `collect` — по событиям прогона
(id запросов редких классов, выборка нормальных, окна) ищет трассы по
`span.messenger.request_id` (и цельные цепочки загрузок), записи журнала по
`request_id` и их переход к трассе, счётчики Prometheus (приём, потери,
события), ряды по `job`, `status/tsdb`, признаки слоёв Collector и
Prometheus, квантили из buckets против журнала. Результат поиска, равный
пределу числа результатов, — отказ «lookup truncated» (код 1): неполный счёт
не выдаётся за полный.
"""
import contextlib
import json
import math
import sys
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from typing import Any
PROMETHEUS = "http://prometheus:9090"
LOKI = "http://loki:3100"
TEMPO = "http://tempo:3200"
COLLECTOR = "http://otel-collector:8888"
READY = (
f"{PROMETHEUS}/-/ready",
f"{LOKI}/ready",
f"{TEMPO}/ready",
f"{COLLECTOR}/metrics",
)
THREADS = 8
# переходы журнал → трасса: потоков на класс (классы идут параллельно)
LINK_THREADS = 2
# одновременных запросов к Tempo (поиски классов и переходы журнал → трасса):
# под восемью поисками и загрузками трасс сразу Tempo с лимитом рабочего
# стенда (640m без swap) завершился посреди выборки metrics-r3 метки baseline
# (design, «Уточнения apply», п. 16); выборка идёт после окон замера
TEMPO_REQUESTS = 3
TEMPO_GATE = threading.BoundedSemaphore(TEMPO_REQUESTS)
TIMEOUT_S = 60
ATTEMPTS = 4
RETRY_PAUSE_S = 1.0
# счётчики по краям окна: шаг — интервал scrape, взгляд назад — был ли ряд
# до окна
STEP_S = 15
LOOKBACK_S = 60
# id в одном поиске Tempo и Loki: регулярное выражение в query string
BATCH = 40
# предел результатов — не меньше 4 × ожидаемого (решение 9)
LIMIT_FACTOR = 4
API_SERVICE = "messenger-api"
REQUEST_ATTR = "span.messenger.request_id"
# у загрузки цепочка — server span API, span producer relay, consumer span
# worker с итогом задания
CHAIN_OUTCOME = {"upload": "ready", "upload_bad": "failed"}
# имена Collector — без суффикса _total у собственных метрик (rule
# telemetry.md); на всякий случай оба варианта
TOTALS = {
"sdk_span_started_sampled": (
'sum(otel_sdk_span_started_total{otel_span_sampling_result='
'"RECORD_AND_SAMPLE"})'
),
"sdk_span_started_dropped": (
'sum(otel_sdk_span_started_total{otel_span_sampling_result="DROP"})'
),
"sdk_span_processed_ok": (
'sum(otel_sdk_processor_span_processed_total{error_type=""})'
),
"sdk_span_processed_lost": (
'sum(otel_sdk_processor_span_processed_total{error_type!=""})'
),
"sdk_span_exported_ok": (
'sum(otel_sdk_exporter_span_exported_total{error_type=""})'
),
"sdk_span_export_failed": (
'sum(otel_sdk_exporter_span_exported_total{error_type!=""})'
),
"sdk_log_processed_ok": (
'sum(otel_sdk_processor_log_processed_total{error_type=""})'
),
"sdk_log_processed_lost": (
'sum(otel_sdk_processor_log_processed_total{error_type!=""})'
),
"sdk_log_exported_ok": (
'sum(otel_sdk_exporter_log_exported_total{error_type=""})'
),
"sdk_log_export_failed": (
'sum(otel_sdk_exporter_log_exported_total{error_type!=""})'
),
"col_accepted_spans": "otelcol_receiver_accepted_spans",
"col_refused_spans": "otelcol_receiver_refused_spans",
"col_failed_spans": "otelcol_receiver_failed_spans",
"col_accepted_logs": "otelcol_receiver_accepted_log_records",
"col_refused_logs": "otelcol_receiver_refused_log_records",
"col_failed_logs": "otelcol_receiver_failed_log_records",
"col_accepted_points": "otelcol_receiver_accepted_metric_points",
"col_refused_points": "otelcol_receiver_refused_metric_points",
"col_sent_spans": "otelcol_exporter_sent_spans",
"col_send_failed_spans": "otelcol_exporter_send_failed_spans",
"col_enqueue_failed_spans": "otelcol_exporter_enqueue_failed_spans",
"col_sent_logs": "otelcol_exporter_sent_log_records",
"col_send_failed_logs": "otelcol_exporter_send_failed_log_records",
"col_enqueue_failed_logs": "otelcol_exporter_enqueue_failed_log_records",
"col_sent_points": "otelcol_exporter_sent_metric_points",
"tail_sampled": (
"otelcol_processor_tail_sampling_global_count_traces_sampled"
'{sampled="true"}'
),
"tail_not_sampled": (
"otelcol_processor_tail_sampling_global_count_traces_sampled"
'{sampled="false"}'
),
"tail_dropped_too_early": (
"otelcol_processor_tail_sampling_sampling_trace_dropped_too_early"
),
"tail_new_traces": (
"otelcol_processor_tail_sampling_new_trace_id_received"
),
"tail_policy_errors": (
"otelcol_processor_tail_sampling_sampling_policy_evaluation_error"
),
"tempo_spans": "sum(tempo_distributor_spans_received_total)",
"tempo_bytes": "sum(tempo_distributor_bytes_received_total)",
"loki_lines": "sum(loki_distributor_lines_received_total)",
"loki_bytes": "sum(loki_distributor_bytes_received_total)",
"loki_sm_bytes": (
"sum(loki_distributor_structured_metadata_bytes_received_total)"
),
"prom_series": "max(prometheus_tsdb_head_series)",
"prom_samples": "sum(prometheus_tsdb_head_samples_appended_total)",
}
# счётчики, приращение которых нужно за чистое окно (приём, решение 9)
CLEAN = (
"col_accepted_spans",
"col_accepted_logs",
"col_accepted_points",
"tempo_spans",
"tempo_bytes",
"loki_lines",
"loki_bytes",
"loki_sm_bytes",
"prom_samples",
"prom_series",
)
GAUGES = frozenset({"prom_series"})
CONTENT_ROUTE = "/api/v1/attachments/{attachment_id}/content"
MESSAGES_ROUTE = "/api/v1/chats/{chat_id}/messages"
HTTP_COUNT = "http_server_request_duration_seconds_count"
HTTP_BUCKET = "http_server_request_duration_seconds_bucket"
RARE_METRICS = {
"broken": (
f'sum({HTTP_COUNT}{{http_route="{CONTENT_ROUTE}",'
'http_response_status_code="500"})'
),
"register": (
f'sum({HTTP_COUNT}{{http_route="/api/v1/users",'
'http_request_method="POST",http_response_status_code="201"})'
),
"write_slow": (
f'sum({HTTP_COUNT}{{http_route="{MESSAGES_ROUTE}",'
'http_request_method="POST"}) - '
f'sum({HTTP_BUCKET}{{http_route="{MESSAGES_ROUTE}",'
'http_request_method="POST",le="0.25"})'
),
"upload_bad": (
'sum(messenger_job_duration_seconds_count{messenger_job_outcome='
'"failed"})'
),
"upload": "sum(messenger_job_time_to_ready_seconds_count)",
}
QUANTILES = (0.95, 0.99)
class TruncatedError(Exception):
"""Поиск упёрся в предел числа результатов."""
def collector_name(expr: str) -> str:
"""Имя Collector с и без `_total`: `sum({__name__=~"имя(_total)?", …})`."""
if expr.startswith(("sum(", "max(")):
return expr
name, _, labels = expr.partition("{")
labels = labels.rstrip("}")
matcher = f'__name__=~"{name}(_total)?"'
if labels:
matcher += f",{labels}"
return f"sum({{{matcher}}})"
def fetch(url: str) -> bytes:
"""GET с повтором на обрыв соединения и 5xx: хранилище с 0,5 CPU под
пулом запросов изредка закрывает соединение без ответа. 4xx — сразу.
Запрос к Tempo ждёт места `TEMPO_GATE`; пауза повтора места не держит."""
for attempt in range(1, ATTEMPTS + 1):
gate = (
TEMPO_GATE if url.startswith(TEMPO) else contextlib.nullcontext()
)
try:
with (
gate,
urllib.request.urlopen(url, timeout=TIMEOUT_S) as response,
):
return response.read()
except urllib.error.HTTPError as exc:
if exc.code < 500 or attempt == ATTEMPTS:
raise
except (OSError, urllib.error.URLError):
if attempt == ATTEMPTS:
raise
time.sleep(RETRY_PAUSE_S * attempt)
raise AssertionError("unreachable")
def get(url: str, params: dict[str, Any] | None = None) -> Any:
if params:
url = f"{url}?{urllib.parse.urlencode(params)}"
body = fetch(url)
return json.loads(body) if body[:1] in (b"{", b"[") else body
def exists(url: str) -> bool:
try:
fetch(url)
except urllib.error.HTTPError as exc:
if exc.code == 404:
return False
raise
return True
def wait_ready(timeout: float) -> int:
deadline = time.monotonic() + timeout
pending = list(READY)
while time.monotonic() < deadline:
left = []
for url in pending:
try:
with urllib.request.urlopen(url, timeout=3) as response:
response.read()
if response.status != 200:
left.append(url)
except (OSError, urllib.error.URLError):
left.append(url)
pending = left
if not pending:
print("telemetry storages ready", flush=True)
return 0
time.sleep(1)
print(f"not ready in {timeout:.0f}s: {pending}", flush=True)
return 1
def batches(items: list[str]) -> list[list[str]]:
return [items[i : i + BATCH] for i in range(0, len(items), BATCH)]
def tempo_search(query: str, window: dict[str, float], expected: int) -> Any:
limit = max(20, LIMIT_FACTOR * expected)
data = get(
f"{TEMPO}/api/search",
{
"q": query,
"start": int(window["start"]),
"end": int(window["end"]) + 1,
"limit": limit,
"spss": 10,
},
)
traces = data.get("traces") or []
if len(traces) >= limit:
raise TruncatedError(
f"lookup truncated: tempo {len(traces)} = {limit}",
)
return traces
def request_ids_in(traces: list[dict[str, Any]]) -> set[str]:
found = set()
for item in traces:
for spanset in item.get("spanSets") or [item.get("spanSet") or {}]:
for span in spanset.get("spans") or []:
for attr in span.get("attributes") or []:
if attr.get("key") == "messenger.request_id":
found.add(attr["value"].get("stringValue", ""))
return found
def traces_of(
ids: list[str],
window: dict[str, float],
outcome: str | None = None,
) -> set[str]:
"""Id запросов, у которых в Tempo есть трасса (у `outcome` — цельная
цепочка загрузки с consumer span этого итога и span producer)."""
found: set[str] = set()
for part in batches(ids):
pattern = "|".join(part)
query = f'{{ {REQUEST_ATTR} =~ "{pattern}" }}'
if outcome is not None:
query += (
" && { kind = consumer && span.messenger.job.outcome = "
f'"{outcome}" }} && {{ kind = producer }}'
)
query += f" | select({REQUEST_ATTR})"
found |= request_ids_in(tempo_search(query, window, len(part)))
return found & set(ids)
def loki_records(
ids: list[str],
window: dict[str, float],
) -> dict[str, dict[str, str]]:
"""Запись доступа API по `request_id`: метки потока (поля записи)."""
out: dict[str, dict[str, str]] = {}
for part in batches(ids):
limit = LIMIT_FACTOR * len(part) + 10
query = (
f'{{service_name="{API_SERVICE}"}} | middleware="http" | '
f'request_id=~"{"|".join(part)}"'
)
data = get(
f"{LOKI}/loki/api/v1/query_range",
{
"query": query,
"start": int(window["start"] * 1e9),
"end": int((window["end"] + 1) * 1e9),
"limit": limit,
"direction": "forward",
},
)
streams = data["data"]["result"]
lines = sum(len(stream["values"]) for stream in streams)
if lines >= limit:
raise TruncatedError(f"lookup truncated: loki {lines} = {limit}")
for stream in streams:
labels = stream["stream"]
out[labels.get("request_id", "")] = labels
return {key: value for key, value in out.items() if key in set(ids)}
def trace_exists(trace_id: str) -> bool:
"""Трасса есть и в ней есть span: на неизвестный id Tempo может ответить
и пустым телом (пробный прогон `head` — 12 «найденных» трасс из 12
невыбранных)."""
try:
body = fetch(f"{TEMPO}/api/v2/traces/{trace_id}")
except urllib.error.HTTPError as exc:
if exc.code == 404:
return False
raise
if not body.strip():
return False
data = json.loads(body)
trace = data.get("trace", data)
return bool(trace.get("resourceSpans") or trace.get("batches"))
def class_result(
name: str,
ids: list[str],
window: dict[str, float],
slow_ms: float,
) -> dict[str, Any]:
traced = traces_of(ids, window)
chains = None
if name in CHAIN_OUTCOME and ids:
chains = len(traces_of(ids, window, CHAIN_OUTCOME[name]))
records = loki_records(ids, window) if ids else {}
with_trace = {
rid: labels["trace_id"]
for rid, labels in records.items()
if labels.get("trace_id")
}
sampled = sum(
1
for labels in records.values()
if labels.get("flags", "").isdigit() and int(labels["flags"]) & 1
)
# свой пул: задачи классов уже заняли потоки общего
with ThreadPoolExecutor(LINK_THREADS) as links:
linked_by = dict(
zip(
with_trace,
links.map(trace_exists, with_trace.values()),
strict=True,
),
)
linked = sum(linked_by.values())
# медленные по серверу: длительность записи доступа (та же величина, что
# server span) не меньше порога; по ней решает правило выбора трасс
server_slow = {
rid
for rid, labels in records.items()
if float(labels.get("duration_ms") or 0) >= slow_ms
}
return {
"ids": len(ids),
"traces": len(traced),
"server_slow": len(server_slow),
"server_slow_traces": len(server_slow & traced),
"server_slow_linked": sum(
1 for rid in server_slow if linked_by.get(rid)
),
"chains": chains,
"logs": len(records),
"logs_with_trace": len(with_trace),
"logs_sampled_flag": sampled,
"log_to_trace": linked,
"missing_traces": sorted(set(ids) - traced)[:5],
}
def prom_value(expr: str, at: float) -> float | None:
data = get(f"{PROMETHEUS}/api/v1/query", {"query": expr, "time": at})
result = data["data"]["result"]
if not result:
return None
value = float(result[0]["value"][1])
# histogram_quantile без отсчётов — NaN: в JSON это не число
return None if math.isnan(value) else value
def prom_delta(
expr: str,
start: float,
end: float,
*,
gauge: bool,
) -> dict[str, float | None]:
"""Разность первого отсчёта не раньше начала окна и последнего не позже
конца (шаг 15 с, решение 9); у gauge — последнее значение."""
data = get(
f"{PROMETHEUS}/api/v1/query_range",
{
"query": expr,
"start": start - LOOKBACK_S,
"end": end,
"step": STEP_S,
},
)
result = data["data"]["result"]
if not result:
return {"delta": None, "seconds": None, "last": None}
series = [(float(t), float(v)) for t, v in result[0]["values"]]
inside = [(t, v) for t, v in series if start <= t <= end]
if not inside:
return {"delta": None, "seconds": None, "last": None}
t1, v1 = inside[-1]
if gauge:
return {"delta": None, "seconds": t1 - inside[0][0], "last": v1}
# ряд счётчика, которого до окна не было (Tempo заводит его на первый
# принятый span — у tail через срок решения), начинается с нуля: база —
# ноль в начале окна, а не первый отсчёт внутри (иначе приёма «нет»)
if any(t < start for t, _ in series) and len(inside) >= 2:
t0, v0 = inside[0]
else:
t0, v0 = start, 0.0
if t1 <= t0:
return {"delta": None, "seconds": None, "last": v1}
return {"delta": v1 - v0, "seconds": t1 - t0, "last": v1}
def quantiles(
method: str,
route: str,
window: dict[str, float],
) -> dict[str, float | None]:
seconds = max(1, int(window["end"] - window["start"]))
out: dict[str, float | None] = {}
for q in QUANTILES:
key = f"p{int(q * 100)}"
out[f"{key}_buckets"] = prom_value(
f"histogram_quantile({q}, sum by (le) (increase({HTTP_BUCKET}"
f'{{http_route="{route}",http_request_method="{method}"}}'
f"[{seconds}s])))",
window["end"],
)
data = get(
f"{LOKI}/loki/api/v1/query",
{
"query": (
f"quantile_over_time({q}, "
f'{{service_name="{API_SERVICE}"}} | middleware="http" '
f'| method="{method}" | path="{route}" '
f"| unwrap duration_ms [{seconds}s]) by ()"
),
"time": int(window["end"] * 1e9),
},
)
result = data["data"]["result"]
value = float(result[0]["value"][1]) / 1000 if result else None
out[f"{key}_log"] = (
None if value is None or math.isnan(value) else value
)
return out
def loki_counts(window: dict[str, float]) -> dict[str, int]:
seconds = max(1, int(window["end"] - window["start"]))
data = get(
f"{LOKI}/loki/api/v1/query",
{
"query": (
"sum by (method, path) (count_over_time("
f'{{service_name="{API_SERVICE}"}} | middleware="http" '
f"[{seconds}s]))"
),
"time": int(window["end"] * 1e9),
},
)
return {
f"{item['metric'].get('method', '')} {item['metric'].get('path', '')}": (
int(float(item["value"][1]))
)
for item in data["data"]["result"]
}
def collect(events: dict[str, Any]) -> dict[str, Any]:
started = time.monotonic()
search = events["search"]
run = events["run"]
clean = events["clean"]
out: dict[str, Any] = {"classes": {}}
with ThreadPoolExecutor(THREADS) as pool:
futures = {
name: pool.submit(
class_result,
name,
ids,
search,
float(events.get("slow_ms", 250)),
)
for name, ids in events["classes"].items()
}
sample = events.get("sample_me") or []
sample_future = pool.submit(traces_of, sample, search)
for name, future in futures.items():
out["classes"][name] = future.result()
out["sample_me"] = {
"ids": len(sample),
"traces": len(sample_future.result()),
}
at = search["end"]
names = list(TOTALS)
values = pool.map(
lambda key: prom_value(collector_name(TOTALS[key]), at),
names,
)
out["totals"] = dict(zip(names, values, strict=True))
rare = list(RARE_METRICS)
out["rare_metrics"] = dict(
zip(
rare,
pool.map(lambda key: prom_value(RARE_METRICS[key], at), rare),
strict=True,
),
)
out["clean"] = dict(
zip(
CLEAN,
pool.map(
lambda key: prom_delta(
collector_name(TOTALS[key]),
clean["start"],
clean["end"],
gauge=key in GAUGES,
),
CLEAN,
),
strict=True,
),
)
out["quantiles"] = {
name: quantiles(method, route, clean)
for name, (method, route) in events["routes"].items()
}
out["loki_counts"] = loki_counts(run)
jobs = get(
f"{PROMETHEUS}/api/v1/query",
{"query": 'count by (job) ({__name__=~".+"})', "time": at},
)
out["series_by_job"] = {
item["metric"].get("job", ""): int(float(item["value"][1]))
for item in jobs["data"]["result"]
}
tsdb = get(f"{PROMETHEUS}/api/v1/status/tsdb")["data"]
out["tsdb"] = {
"num_series": tsdb.get("headStats", {}).get("numSeries"),
"top_metrics": tsdb.get("seriesCountByMetricName", [])[:10],
}
config = get(f"{PROMETHEUS}/api/v1/status/config")["data"]["yaml"]
tail = prom_value(
'count({__name__=~"otelcol_processor_tail_sampling_.*"})',
at,
)
out["layers"] = {
"tail_sampling": bool(tail),
"lean_prometheus": "loki_distributor_" in config,
}
out["seconds"] = round(time.monotonic() - started, 1)
return out
def main(argv: list[str]) -> int:
if argv[:1] == ["wait-ready"]:
timeout = float(argv[2]) if argv[1:2] == ["--timeout"] else 90.0
return wait_ready(timeout)
if argv[:1] == ["collect"] and len(argv) == 3:
with open(argv[1], encoding="utf-8") as file:
events = json.load(file)
try:
result = collect(events)
except TruncatedError as exc:
print(str(exc), flush=True)
return 1
with open(argv[2], "w", encoding="utf-8") as file:
json.dump(result, file, ensure_ascii=False, indent=1)
print(f"lookup done in {result['seconds']}s", flush=True)
return 0
print(__doc__, flush=True)
return 2
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))