Files
mtusi2026lab6/bench/telemetry_cost.py
T
ricoromaandClaude Opus 5.5 56132c77e9 fix: address track1-telemetry-cost verify findings
Base accepting the saturated full-r3 run on the user's 2026-10-31 host
warning rule instead of the swap decision and ask whether to re-shoot;
complete the report section with limits, metric freshness at 30 s and
per-class and per-component tables; correct derived numbers and the
quantile error wording; align H6 in design with the report; fix the
quantile_error docstring; park S1 and S4 as later proposals.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 06:12:48 +03:00

2319 lines
81 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Профиль цены телеметрии без docker: варианты, план, разбор, вердикты.
Design change `track1-telemetry-cost`, решения 2, 6, 7, 9–12; оркестратор —
`bench/run.py` (профили `telemetry`, `telemetry_quick`). Только stdlib, `src`
не импортирует. Здесь — таблица вариантов, план кругами, разбор строк k6
(`k6/telemetry.js`), сэмплера `cost`, обхода метаданных, выборки
(`telemetry_lookup.py`) и размера volume; окна; CPU на запрос, приём, рост на
диске, потери по звеньям, доли классов редких событий, точность квантилей;
строки `runs.csv`, `components.csv`, `classes.csv`; правило шума, точный
биномиальный интервал, вердикты H1–H8 и B1–B5, пометки; `summary.md` и четыре
SVG; проверки конфигураций до стенда (слой Collector, копия Prometheus) и
записей о старте процессов.
uv run python bench/telemetry_cost.py --selftest
"""
import json
import math
import random
import statistics
import sys
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import overload
import plots
import reconcile
import scaling
ROOT = Path(__file__).resolve().parents[1]
COLLECTOR_CONFIG = "infra/otel/collector.yaml"
PROMETHEUS_CONFIG = "infra/otel/prometheus/prometheus.yml"
TAIL_TYPICAL = "bench/otel/tail-typical.yaml"
TAIL_BALANCED = "bench/otel/tail-balanced.yaml"
PROMETHEUS_LEAN = "bench/otel/prometheus-lean.yml"
LEAN_BEGIN = "# bench-lean: begin"
LEAN_END = "# bench-lean: end"
# пути конфигураций в контейнерах (docker-compose.yml)
COLLECTOR_TARGET = "/etc/otelcol-contrib/config.yaml"
LAYER_TARGET = "/etc/otelcol-contrib/layer.yaml"
PROMETHEUS_TARGET = "/etc/prometheus/prometheus.yml"
# сроки ожидания решения выбора трасс в Collector, с (decision_wait слоёв)
DECISION_WAIT_S = {TAIL_TYPICAL: 30, TAIL_BALANCED: 10}
SCRAPE_S = 15
EXPORT_MARGIN_S = 5
DECISION_MARGIN_S = 2
# классы запросов (решение 8); лёгкие — для цены
LIGHT = ("read", "write", "me", "card")
CLASSES = (
"read",
"write",
"me",
"card",
"download",
"upload",
"upload_bad",
"broken",
"register",
)
OK_STATUS = {
"read": 200,
"write": 201,
"me": 200,
"card": 200,
"download": 200,
"upload": 202,
"upload_bad": 202,
"broken": 500,
"register": 201,
}
# классы редких событий: что ищется в хранилищах
RARE = ("broken", "write_slow", "register", "upload_bad", "upload")
# редкие события, которые выбор в голове сохраняет с долей около 0,1
HEAD_CLASSES = ("broken", "write_slow", "register", "upload_bad")
# сохранение, которое критерии tail и balanced требуют полным: ошибки,
# медленные по серверу записи (≥ порога по `duration_ms` записи доступа —
# той же величине, что server span) и цепочки неудачных загрузок.
# Регистрация — редкий, но быстрый маршрут (~85 мс на стенде, а не 0,3–1 с,
# как ждал design): у tail и balanced она сохраняется с долей остальных, это
# и есть «что теряется» (design, «Уточнения apply»)
MUST_KEEP = ("broken", "write_slow", "upload_bad")
ROLE_GROUPS = {
"app": ("api", "worker", "relay"),
"telemetry": ("otel-collector", "prometheus", "loki", "tempo"),
}
TELEMETRY_ROLES = ROLE_GROUPS["telemetry"]
COMPONENT_FIELDS = [
"run",
"variant",
"repeat",
"role",
"name",
"cpu_cores",
"throttle",
"mem_max_mib",
"peak_mib",
"anon_max_mib",
]
CLASS_FIELDS = [
"run",
"variant",
"repeat",
"class",
"client",
"traces",
"chains",
"logs",
"logs_with_trace",
"logs_sampled_flag",
"log_to_trace",
"metric",
"server_slow",
"server_slow_traces",
"server_slow_linked",
]
LATENCY_KEYS = ("n", "errors", "p50", "p95", "p99")
INGEST = (
"col_accepted_spans",
"col_accepted_logs",
"col_accepted_points",
"tempo_spans",
"tempo_bytes",
"loki_lines",
"loki_bytes",
"loki_sm_bytes",
"prom_samples",
)
DISKS = ("prometheus", "loki", "tempo")
LOSS_KEYS = (
"sdk_span_lost",
"sdk_log_lost",
"col_span_lost",
"col_log_lost",
"col_point_refused",
"tail_dropped_too_early",
)
RUN_FIELDS = [
"run",
"variant",
"repeat",
"order",
"warm_s",
"clean_s",
"fault_s",
*(f"{name}_{key}" for name in ("light", *CLASSES) for key in LATENCY_KEYS),
"replica_cpu_ms_per_req",
"replica_cpu_cores",
"app_cpu_cores",
"telemetry_cpu_cores",
"rest_cpu_cores",
*(
f"{role.replace('-', '_')}_{key}"
for role in TELEMETRY_ROLES
for key in ("cpu", "throttle", "mem_max_mib", "peak_mib")
),
*(f"{name}_per_s" for name in INGEST),
"prom_series",
*(f"disk_{name}_bytes" for name in DISKS),
"sdk_spans_sampled",
"sdk_spans_dropped",
"sdk_spans_exported",
"col_spans_accepted",
"col_spans_sent",
"tempo_spans_total",
"tail_sampled",
"tail_not_sampled",
*LOSS_KEYS,
"spans_gap",
"ttr_p50_s",
"ttr_p95_s",
"accepted",
"ready",
"bad_accepted",
"bad_failed",
"slow_writes",
"dropped",
*(
f"q_{route}_{q}_{src}"
for route in ("read", "write")
for q in ("p95", "p99")
for src in ("buckets", "log")
),
"flags",
]
# пометки, при которых код команды в конце метки ненулевой (решение 11)
FATAL_FLAGS = ("lost",)
# калибровка (решение 8): CPU реплики в чистом окне, предел Collector,
# Tempo и worker, p95 времени до ready
CAL_REPLICA = (0.30, 0.60)
CAL_LIMITED = 0.70
CAL_TTR_P95_S = 5.0
Z_ALPHA = 0.05
# ------------------------------------------------------------- варианты
def variants(head_ratio: float, metrics_interval: float) -> dict[str, Any]:
"""Таблица вариантов (решение 2): настройки SDK трёх процессов, слой
Collector и конфигурация Prometheus. Loki и Tempo — без изменений."""
full = {
"telemetry": True,
"ratio": 1.0,
"buckets": "default",
"interval": 15,
"collector_layer": None,
"prometheus": None,
}
return {
"off": {"telemetry": False},
"full": full,
"full-aa": dict(full),
"head": {**full, "ratio": head_ratio},
"tail": {**full, "collector_layer": TAIL_TYPICAL},
"metrics": {
**full,
"interval": metrics_interval,
"buckets": "coarse",
"prometheus": PROMETHEUS_LEAN,
},
"balanced": {
**full,
"collector_layer": TAIL_BALANCED,
"prometheus": PROMETHEUS_LEAN,
},
}
def sdk_env(variant: Mapping[str, Any]) -> dict[str, str]:
"""Окружение `TELEMETRY__*` реплик, worker и relay варианта (без
включения и адреса — их даёт `telemetry.compose.yml`)."""
if not variant.get("telemetry"):
return {}
return {
"TELEMETRY__TRACE_SAMPLE_RATIO": f"{variant['ratio']:g}",
"TELEMETRY__METRIC_BUCKETS": variant["buckets"],
"TELEMETRY__METRIC_EXPORT_INTERVAL_SECONDS": f"{variant['interval']:g}",
}
def wait_after(variant: Mapping[str, Any]) -> float:
"""Ожидание отправок после нагрузки (решение 9): последняя отправка
метрик + scrape и решение выбора трасс доходят до хранилищ раньше
выборки."""
if not variant.get("telemetry"):
return 0.0
metrics = variant["interval"] + SCRAPE_S + EXPORT_MARGIN_S
layer = variant.get("collector_layer")
# после решения span ещё уходят в Tempo, а счётчик Tempo доходит до
# Prometheus через scrape (пробный tail без scrape — расхождение 8 %)
decision = (
DECISION_WAIT_S[layer] + DECISION_MARGIN_S + SCRAPE_S + EXPORT_MARGIN_S
if layer
else 0
)
return float(max(metrics, decision))
def variant_compose(variant: Mapping[str, Any], root: Path) -> str:
"""`.work/variant.compose.yml` прогона (JSON — подмножество YAML):
окружение SDK трёх процессов, у Collector — второй `--config` и bind слоя,
у Prometheus — bind своей конфигурации на тот же путь (compose сливает
volumes по пути в контейнере)."""
services: dict[str, Any] = {}
env = sdk_env(variant)
for service in ("api", "worker", "relay"):
if env:
services[service] = {"environment": dict(env)}
layer = variant.get("collector_layer")
if layer:
services["otel-collector"] = {
"command": [
f"--config={COLLECTOR_TARGET}",
f"--config={LAYER_TARGET}",
],
"volumes": [bind(root / layer, LAYER_TARGET)],
}
lean = variant.get("prometheus")
if lean:
services["prometheus"] = {
"volumes": [bind(root / lean, PROMETHEUS_TARGET)],
}
return json.dumps({"services": services}, indent=1) + "\n"
def bind(source: Path, target: str) -> dict[str, Any]:
return {
"type": "bind",
"source": str(source),
"target": target,
"read_only": True,
"bind": {"create_host_path": False},
}
def plan_runs(
names: Iterable[str],
repeats: int,
seed: int,
) -> list[dict[str, Any]]:
"""Круги с перестановкой от зерна; круг не начинается с варианта, которым
кончился прошлый: повтор одного варианта подряд не идёт."""
items = list(names)
plan: list[dict[str, Any]] = []
previous = None
for round_no in range(1, repeats + 1):
order = list(items)
random.Random(seed + round_no).shuffle(order)
if len(order) > 1 and order[0] == previous:
order[0], order[1] = order[1], order[0]
for name in order:
plan.append(
{
"run": f"{name}-r{round_no}",
"variant": name,
"repeat": round_no,
"order": len(plan) + 1,
},
)
previous = order[-1] if order else None
return plan
# ------------------------------------------------- проверки конфигураций
def lean_problems(lean: str, base: str) -> list[str]:
"""Копия Prometheus без размеченных блоков совпадает с рабочей построчно
(решение 7)."""
kept: list[str] = []
inside = False
for number, line in enumerate(lean.splitlines(), start=1):
mark = line.strip()
if mark == LEAN_BEGIN:
if inside:
return [f"line {number}: nested {LEAN_BEGIN}"]
inside = True
continue
if mark == LEAN_END:
if not inside:
return [f"line {number}: {LEAN_END} without begin"]
inside = False
continue
if not inside:
kept.append(line)
if inside:
return [f"{LEAN_BEGIN} is not closed"]
problems = []
base_lines = base.splitlines()
for number, (a, b) in enumerate(zip(kept, base_lines, strict=False), 1):
if a != b:
problems.append(
f"outside bench-lean blocks line {number} differs from "
f"{PROMETHEUS_CONFIG}: {a!r} != {b!r}",
)
break
if not problems and len(kept) != len(base_lines):
problems.append(
f"outside bench-lean blocks {len(kept)} lines, "
f"{PROMETHEUS_CONFIG} {len(base_lines)}",
)
return problems
def first_difference(a: Any, b: Any, path: str = "") -> str | None:
if isinstance(a, dict) and isinstance(b, dict):
for key in sorted(set(a) | set(b)):
if key not in a or key not in b:
return f"{path}.{key}".lstrip(".")
found = first_difference(a[key], b[key], f"{path}.{key}")
if found is not None:
return found
return None
return None if a == b else (path.lstrip(".") or "<root>")
def collector_layer_problems(
base: Mapping[str, Any],
layered: Mapping[str, Any],
) -> list[str]:
"""Итоговая конфигурация со слоем отличается от рабочей только
процессором `tail_sampling` и списком процессоров pipeline трасс — базовым
с `tail_sampling` перед `batch` (решение 6)."""
layered = json.loads(json.dumps(layered))
problems = []
processors = layered.get("processors") or {}
if "tail_sampling" not in processors:
problems.append("layer has no processors.tail_sampling")
processors.pop("tail_sampling", None)
traces = layered.get("service", {}).get("pipelines", {}).get("traces", {})
base_list = list(base["service"]["pipelines"]["traces"]["processors"])
if "batch" in base_list:
expected = list(base_list)
expected.insert(expected.index("batch"), "tail_sampling")
else:
expected = [*base_list, "tail_sampling"]
if traces.get("processors") != expected:
problems.append(
f"traces processors {traces.get('processors')}, expected "
f"{expected}",
)
traces["processors"] = base_list
diff = first_difference(dict(base), layered)
if diff is not None:
problems.append(f"layer changes {diff}")
return problems
STARTED = {
"api": "Started API",
"worker": "Started worker",
"relay": "Started relay",
}
def startup_problems(
records_by_process: Mapping[str, list[dict[str, Any]]],
variant: Mapping[str, Any],
) -> list[str]:
"""Записи о старте реплик, worker и relay против варианта (решение 11):
включена ли телеметрия, доля, набор границ, интервал."""
problems = []
for process, records in records_by_process.items():
kind = process.split(":", 1)[0]
event = STARTED[kind]
started = [r for r in records if r.get("event") == event]
if not started:
problems.append(f"{process}: no «{event}» record")
continue
record = started[-1]
expected: dict[str, Any] = {
"telemetry_enabled": bool(variant.get("telemetry")),
}
if variant.get("telemetry"):
expected.update(
{
"telemetry_trace_sample_ratio": float(variant["ratio"]),
"telemetry_metric_buckets": variant["buckets"],
"telemetry_metric_export_interval_seconds": float(
variant["interval"],
),
},
)
for key, value in expected.items():
seen = record.get(key)
if isinstance(value, float):
ok = isinstance(seen, int | float) and math.isclose(
float(seen),
value,
)
else:
ok = seen == value
if not ok:
problems.append(
f"{process}: {key} {seen!r}, expected {value!r}",
)
return problems
def layer_problems(
layers: Mapping[str, bool],
variant: Mapping[str, Any],
) -> list[str]:
"""Слои Collector и Prometheus по выборке (решение 9)."""
problems = []
want_tail = bool(variant.get("collector_layer"))
if bool(layers.get("tail_sampling")) != want_tail:
problems.append(
f"tail sampling series {layers.get('tail_sampling')}, "
f"expected {want_tail}",
)
want_lean = bool(variant.get("prometheus"))
if bool(layers.get("lean_prometheus")) != want_lean:
problems.append(
f"lean Prometheus config {layers.get('lean_prometheus')}, "
f"expected {want_lean}",
)
return problems
# ------------------------------------------------------------- разбор
@dataclass(frozen=True)
class Req:
cls: str
sent_ms: float
duration_ms: float
status: int
request_id: str
instance: str
attachment_id: str
@property
def answered(self) -> float:
return (self.sent_ms + self.duration_ms) / 1000
def parse_k6(text: str) -> tuple[float | None, list[Req]]:
"""Строки `t-t0,<мс>` и `t,<класс>,<отправка>,<длительность>,<статус>,
<X-Request-Id>,<X-Instance-Id>,<id>`."""
t0 = None
out = []
for raw in text.splitlines():
parts = raw.strip().split(",")
if parts[0] == "t-t0" and len(parts) == 2:
t0 = float(parts[1]) / 1000
elif parts[0] == "t" and len(parts) == 8:
out.append(
Req(
parts[1],
float(parts[2]),
float(parts[3]),
int(parts[4]),
parts[5],
parts[6],
parts[7],
),
)
return t0, out
def window_bounds(
t0: float,
windows: tuple[int, int, int],
) -> dict[str, tuple[float, float]]:
warm, clean, fault = windows
return {
"warm": (t0, t0 + warm),
"clean": (t0 + warm, t0 + warm + clean),
"fault": (t0 + warm + clean, t0 + warm + clean + fault),
"run": (t0, t0 + warm + clean + fault),
}
def inside(reqs: Iterable[Req], w: tuple[float, float]) -> list[Req]:
return [r for r in reqs if w[0] * 1000 <= r.sent_ms < w[1] * 1000]
def latency(reqs: list[Req], classes: Iterable[str]) -> dict[str, Any]:
names = set(classes)
chosen = [r for r in reqs if r.cls in names]
ok = [r.duration_ms for r in chosen if r.status == OK_STATUS[r.cls]]
return {
"n": len(chosen),
"errors": len(chosen) - len(ok),
"p50": rounded(reconcile.percentile(ok, 50), 2),
"p95": rounded(reconcile.percentile(ok, 95), 2),
"p99": rounded(reconcile.percentile(ok, 99), 2),
}
def rounded(value: float | None, digits: int = 3) -> float | None:
return None if value is None else round(value, digits)
def rare_ids(reqs: list[Req], slow_ms: float) -> dict[str, list[str]]:
"""Id запросов классов редких событий (по клиенту): 500 вложения без
объекта, медленные записи, регистрации, принятые загрузки `bad` и
обычные."""
def ids(cls: str, status: int) -> list[str]:
return [
r.request_id
for r in reqs
if r.cls == cls and r.status == status and r.request_id != "-"
]
return {
"broken": ids("broken", 500),
"write_slow": [
r.request_id
for r in reqs
if r.cls == "write"
and r.status == 201
and r.duration_ms >= slow_ms
and r.request_id != "-"
],
"register": ids("register", 201),
"upload_bad": ids("upload_bad", 202),
"upload": ids("upload", 202),
}
def parse_du(text: str) -> dict[str, int]:
"""`du -sb /v/<имя>` по строкам: имя volume → байт."""
out = {}
for line in text.splitlines():
parts = line.split()
if len(parts) == 2 and parts[0].isdigit():
out[parts[1].rstrip("/").rsplit("/", 1)[-1]] = int(parts[0])
return out
@dataclass
class Sources:
"""Всё снятое за прогон."""
run: Mapping[str, Any]
variant: Mapping[str, Any]
windows: tuple[int, int, int]
t0: float
reqs: list[Req]
dropped: int
sampler_text: str
containers: Mapping[str, Mapping[str, str]]
metas: Mapping[str, Any]
lookup: Mapping[str, Any] | None
du: Mapping[str, int]
slow_ms: float
not_drained: bool = False
storage_down: bool = False
oom: bool = False
swapouts: int | None = None
def components(src: Sources) -> list[dict[str, Any]]:
"""CPU, троттлинг и память каждого контейнера стенда в чистом окне."""
w0, w1 = window_bounds(src.t0, src.windows)["clean"]
samples = scaling.parse_samples(src.sampler_text)
mp = overload.parse_mp(src.sampler_text)
rows = []
for (_, ident), points in sorted(samples.cg.items()):
info = src.containers.get(ident)
if not info:
continue
lives = overload.split_lives(points)
cores = overload.role_cores([lives], w0, w1)
in_window = [p for p in points if w0 * 1e9 <= p[0] <= w1 * 1e9]
peaks = mp.get(ident, [])
rows.append(
{
"run": src.run["run"],
"variant": src.run["variant"],
"repeat": src.run["repeat"],
"role": info["role"],
"name": info["name"],
"cpu_cores": cores,
"throttle": overload.throttle_share(lives, w0, w1),
"mem_max_mib": (
round(max(p[5] for p in in_window) / 2**20, 1)
if in_window
else None
),
"peak_mib": (
round(max(p[1] for p in peaks) / 2**20, 1)
if peaks
else None
),
"anon_max_mib": (
round(max(p[2] for p in peaks) / 2**20, 1)
if peaks
else None
),
},
)
return rows
def group_cores(rows: list[dict[str, Any]]) -> dict[str, float | None]:
out: dict[str, float] = {"app": 0.0, "telemetry": 0.0, "rest": 0.0}
seen = False
for row in rows:
cores = row["cpu_cores"]
if cores is None or row["role"] in ("k6", "sampler"):
continue
seen = True
group = next(
(g for g, roles in ROLE_GROUPS.items() if row["role"] in roles),
"rest",
)
out[group] += cores
if not seen:
return dict.fromkeys(out)
return {key: round(value, 4) for key, value in out.items()}
def num(value: Any) -> float | None:
return reconcile.num(value)
def zero(value: Any) -> float:
return num(value) or 0.0
def losses(totals: Mapping[str, Any]) -> dict[str, float]:
"""Потери по звеньям (решение 9): SDK — отброшенное процессором и
неудачные отправки; Collector — отказ приёма, ошибки и отказы отправки.
Отброшенное выборкой (в голове и решения Collector) сюда не входит."""
return {
"sdk_span_lost": zero(totals.get("sdk_span_processed_lost"))
+ zero(totals.get("sdk_span_export_failed")),
"sdk_log_lost": zero(totals.get("sdk_log_processed_lost"))
+ zero(totals.get("sdk_log_export_failed")),
"col_span_lost": sum(
zero(totals.get(key))
for key in (
"col_refused_spans",
"col_failed_spans",
"col_send_failed_spans",
"col_enqueue_failed_spans",
)
),
"col_log_lost": sum(
zero(totals.get(key))
for key in (
"col_refused_logs",
"col_failed_logs",
"col_send_failed_logs",
"col_enqueue_failed_logs",
)
),
"col_point_refused": zero(totals.get("col_refused_points")),
"tail_dropped_too_early": zero(totals.get("tail_dropped_too_early")),
}
def spans_gap(totals: Mapping[str, Any]) -> float | None:
"""Доля, на которую span не сходятся по звеньям: SDK отправил → Collector
принял, Collector отправил → Tempo принял (наибольшая из двух)."""
pairs = (
("sdk_span_exported_ok", "col_accepted_spans"),
("col_sent_spans", "tempo_spans"),
)
gaps = []
for a, b in pairs:
va, vb = num(totals.get(a)), num(totals.get(b))
if va is None or vb is None or not va:
continue
gaps.append(abs(va - vb) / va)
return round(max(gaps), 4) if gaps else None
def time_to_ready(
reqs: list[Req],
metas: Mapping[str, Any],
) -> tuple[float | None, float | None]:
values = []
for r in reqs:
meta = metas.get(r.attachment_id)
if (
r.cls == "upload"
and r.status == 202
and meta is not None
and meta.status == "ready"
and meta.updated_at is not None
):
values.append(max(0.0, meta.updated_at - r.answered))
return (
rounded(reconcile.percentile(values, 50), 3),
rounded(reconcile.percentile(values, 95), 3),
)
def run_row(
src: Sources,
comps: list[dict[str, Any]],
) -> dict[str, Any]:
bounds = window_bounds(src.t0, src.windows)
clean = inside(src.reqs, bounds["clean"])
row: dict[str, Any] = {
key: src.run[key] for key in ("run", "variant", "repeat", "order")
}
row.update(
dict(zip(("warm_s", "clean_s", "fault_s"), src.windows, strict=True)),
)
for name, classes in (("light", LIGHT), *((c, (c,)) for c in CLASSES)):
for key, value in latency(clean, classes).items():
row[f"{name}_{key}"] = value
replicas = [c for c in comps if c["role"] == "api"]
replica_cores = (
sum(c["cpu_cores"] for c in replicas if c["cpu_cores"] is not None)
if replicas
else None
)
answered = sum(1 for r in clean if r.instance != "-")
clean_s = src.windows[1]
row["replica_cpu_cores"] = rounded(replica_cores, 4)
row["replica_cpu_ms_per_req"] = (
round(replica_cores * clean_s * 1000 / answered, 4)
if replica_cores is not None and answered
else None
)
groups = group_cores(comps)
for group in ("app", "telemetry", "rest"):
row[f"{group}_cpu_cores"] = groups[group]
for role in TELEMETRY_ROLES:
found = [c for c in comps if c["role"] == role]
prefix = role.replace("-", "_")
item = found[0] if found else {}
row[f"{prefix}_cpu"] = item.get("cpu_cores")
row[f"{prefix}_throttle"] = item.get("throttle")
row[f"{prefix}_mem_max_mib"] = item.get("mem_max_mib")
row[f"{prefix}_peak_mib"] = item.get("peak_mib")
lookup = src.lookup or {}
clean_counts = lookup.get("clean", {})
for name in INGEST:
item = clean_counts.get(name) or {}
delta, seconds = item.get("delta"), item.get("seconds")
row[f"{name}_per_s"] = (
round(delta / seconds, 3)
if delta is not None and seconds
else None
)
row["prom_series"] = (clean_counts.get("prom_series") or {}).get("last")
for name in DISKS:
row[f"disk_{name}_bytes"] = src.du.get(name)
totals = lookup.get("totals", {})
row["sdk_spans_sampled"] = totals.get("sdk_span_started_sampled")
row["sdk_spans_dropped"] = totals.get("sdk_span_started_dropped")
row["sdk_spans_exported"] = totals.get("sdk_span_exported_ok")
row["col_spans_accepted"] = totals.get("col_accepted_spans")
row["col_spans_sent"] = totals.get("col_sent_spans")
row["tempo_spans_total"] = totals.get("tempo_spans")
row["tail_sampled"] = totals.get("tail_sampled")
row["tail_not_sampled"] = totals.get("tail_not_sampled")
lost = losses(totals) if src.lookup else {}
for key in LOSS_KEYS:
row[key] = lost.get(key)
row["spans_gap"] = spans_gap(totals) if src.lookup else None
row["ttr_p50_s"], row["ttr_p95_s"] = time_to_ready(src.reqs, src.metas)
accepted = [r for r in src.reqs if r.cls == "upload" and r.status == 202]
bad = [r for r in src.reqs if r.cls == "upload_bad" and r.status == 202]
def final(items: list[Req], status: str, code: str | None) -> int:
count = 0
for r in items:
meta = src.metas.get(r.attachment_id)
if meta is None or meta.status != status:
continue
if code is not None and meta.error_code != code:
continue
count += 1
return count
row["accepted"] = len(accepted)
row["ready"] = final(accepted, "ready", None)
row["bad_accepted"] = len(bad)
row["bad_failed"] = final(bad, "failed", "invalid_image")
row["slow_writes"] = len(rare_ids(src.reqs, src.slow_ms)["write_slow"])
row["dropped"] = src.dropped
for route in ("read", "write"):
values = (lookup.get("quantiles") or {}).get(route) or {}
for q in ("p95", "p99"):
for source in ("buckets", "log"):
row[f"q_{route}_{q}_{source}"] = rounded(
num(values.get(f"{q}_{source}")),
5,
)
row["flags"] = " ".join(run_flags(src, row))
return row
def run_flags(src: Sources, row: Mapping[str, Any]) -> list[str]:
flags = []
if src.dropped:
flags.append("dropped")
if src.swapouts:
flags.append("host_swap")
if src.storage_down:
flags.append("storage_down")
if src.oom:
flags.append("oom")
if src.not_drained:
flags.append("not_drained")
if (
row["ready"] != row["accepted"]
or row["bad_failed"] != row["bad_accepted"]
):
flags.append("lost")
if src.lookup and any(row.get(key) for key in LOSS_KEYS):
flags.append("telemetry_loss")
if src.run["variant"] == "full" and calibration_problems(row):
flags.append("saturated")
return flags
def calibration_problems(row: Mapping[str, Any]) -> list[str]:
"""Границы калибровки решения 8 у варианта `full`."""
problems = []
cores = num(row.get("replica_cpu_cores"))
if cores is not None:
share = cores / 2
if not CAL_REPLICA[0] <= share <= CAL_REPLICA[1]:
problems.append(f"replica CPU {share:.0%} of 1 CPU (30–60 %)")
for key, limit_cpus in (
("otel_collector_cpu", 0.5),
("tempo_cpu", 0.5),
):
value = num(row.get(key))
if value is not None and value / limit_cpus >= CAL_LIMITED:
problems.append(f"{key} {value / limit_cpus:.0%} of its limit")
if row.get("dropped"):
problems.append(f"dropped iterations {row['dropped']}")
ttr = num(row.get("ttr_p95_s"))
if ttr is not None and ttr >= CAL_TTR_P95_S:
problems.append(f"time to ready p95 {ttr} s")
return problems
def class_rows(src: Sources) -> list[dict[str, Any]]:
ids = rare_ids(src.reqs, src.slow_ms)
lookup = src.lookup or {}
found = lookup.get("classes", {})
metrics = lookup.get("rare_metrics", {})
rows = []
for name in RARE:
item = found.get(name, {})
rows.append(
{
"run": src.run["run"],
"variant": src.run["variant"],
"repeat": src.run["repeat"],
"class": name,
"client": len(ids[name]),
"traces": item.get("traces"),
"chains": item.get("chains"),
"logs": item.get("logs"),
"logs_with_trace": item.get("logs_with_trace"),
"logs_sampled_flag": item.get("logs_sampled_flag"),
"log_to_trace": item.get("log_to_trace"),
"metric": metrics.get(name),
"server_slow": item.get("server_slow"),
"server_slow_traces": item.get("server_slow_traces"),
"server_slow_linked": item.get("server_slow_linked"),
},
)
return rows
def events_for_lookup(
src_reqs: list[Req],
t0: float,
windows: tuple[int, int, int],
*,
slow_ms: float,
sample_me: int,
seed: int,
collected_at: float,
) -> dict[str, Any]:
"""Вход `telemetry_lookup.py collect` (решение 9): id редких классов,
выборка `GET /users/me` чистого окна, окна."""
bounds = window_bounds(t0, windows)
me = [
r.request_id
for r in inside(src_reqs, bounds["clean"])
if r.cls == "me" and r.status == 200
]
random.Random(seed).shuffle(me)
return {
"run": {"start": bounds["run"][0], "end": bounds["run"][1]},
"clean": {"start": bounds["clean"][0], "end": bounds["clean"][1]},
"search": {"start": t0 - 60, "end": collected_at},
"classes": rare_ids(src_reqs, slow_ms),
"slow_ms": slow_ms,
"sample_me": me[:sample_me],
"routes": {
"read": ["GET", "/api/v1/chats/{chat_id}/messages"],
"write": ["POST", "/api/v1/chats/{chat_id}/messages"],
},
}
# ------------------------------------------------------ статистика
def binom_cdf(k: int, n: int, p: float) -> float:
if p <= 0:
return 1.0
if p >= 1:
return 1.0 if k >= n else 0.0
return sum(
math.exp(
math.lgamma(n + 1)
- math.lgamma(i + 1)
- math.lgamma(n - i + 1)
+ i * math.log(p)
+ (n - i) * math.log1p(-p),
)
for i in range(k + 1)
)
def binom_interval(
k: int,
n: int,
alpha: float = Z_ALPHA,
) -> tuple[float, float]:
"""Точный (Клоппер — Пирсон) интервал доли k/n уровня 1 − alpha."""
if n <= 0:
return (0.0, 1.0)
def solve(f: Any) -> float:
lo, hi = 0.0, 1.0
for _ in range(60):
mid = (lo + hi) / 2
if f(mid):
lo = mid
else:
hi = mid
return (lo + hi) / 2
lower = (
0.0
if k == 0
else solve(lambda p: 1 - binom_cdf(k - 1, n, p) < alpha / 2)
)
upper = 1.0 if k == n else solve(lambda p: binom_cdf(k, n, p) > alpha / 2)
return (lower, upper)
def values(rows: Iterable[Mapping[str, Any]], key: str) -> list[float]:
return [v for v in (num(r.get(key)) for r in rows) if v is not None]
def med(rows: Iterable[Mapping[str, Any]], key: str) -> float | None:
found = values(rows, key)
return statistics.median(found) if found else None
def by_variant(
rows: Iterable[Mapping[str, Any]],
) -> dict[str, list[Mapping[str, Any]]]:
out: dict[str, list[Mapping[str, Any]]] = {}
for row in rows:
out.setdefault(row["variant"], []).append(row)
return out
def noise(groups: Mapping[str, list[Mapping[str, Any]]], key: str) -> float:
"""Шум величины — |медиана full − медиана full-aa| (решение 12)."""
a, b = (
med(groups.get("full", []), key),
med(groups.get("full-aa", []), key),
)
if a is None or b is None:
return 0.0
return abs(a - b)
def effect(
groups: Mapping[str, list[Mapping[str, Any]]],
variant: str,
reference: str,
key: str,
) -> bool | None:
"""Эффект: разбросы повторов не пересекаются и |Δ медианы| больше шума."""
a, b = (
values(groups.get(variant, []), key),
values(
groups.get(reference, []),
key,
),
)
if not a or not b:
return None
apart = max(a) < min(b) or max(b) < min(a)
delta = abs(statistics.median(a) - statistics.median(b))
return apart and delta > noise(groups, key)
class Verdict:
"""Условия гипотезы или критерия: «нет данных», если нечем проверить
условие; «нет», если нарушено (причина названа)."""
def __init__(self, name: str) -> None:
self.name = name
self.failed: list[str] = []
self.missing: list[str] = []
self.notes: list[str] = []
def need(self, label: str, ok: bool | None, detail: str = "") -> None:
text = f"{label}{': ' + detail if detail else ''}"
if ok is None:
self.missing.append(text)
elif ok:
self.notes.append(text)
else:
self.failed.append(text)
@property
def result(self) -> str:
if self.failed:
return "нет"
if self.missing:
return "нет данных"
return "подтверждено"
def render(self) -> str:
parts = [f"**{self.name}: {self.result}.**"]
if self.failed:
parts.append("Нарушено: " + "; ".join(self.failed) + ".")
if self.missing:
parts.append("Нет данных: " + "; ".join(self.missing) + ".")
if self.notes:
parts.append("Выполнено: " + "; ".join(self.notes) + ".")
return " ".join(parts)
def fmt(value: float | None, digits: int = 2) -> str:
return "—" if value is None else f"{value:.{digits}f}"
def ratio(a: float | None, b: float | None) -> float | None:
return None if a is None or not b else a / b
def class_share(
classes: list[Mapping[str, Any]],
variant: str,
name: str,
field: str = "traces",
total_field: str = "client",
) -> tuple[int, int] | None:
"""Сумма найденного и всего (по умолчанию — счёт клиента) класса по
повторам варианта."""
rows = [
r for r in classes if r["variant"] == variant and r["class"] == name
]
if not rows or any(
num(r.get(key)) is None for r in rows for key in (field, total_field)
):
return None
found = int(sum(zero(r.get(field)) for r in rows))
total = int(sum(zero(r.get(total_field)) for r in rows))
return found, total
def kept(
classes: list[Mapping[str, Any]],
variant: str,
name: str,
) -> tuple[int, int] | None:
"""Сохранённые трассы класса, который критерии требуют полным: у
медленных записей — среди медленных по серверу, у неудачных загрузок —
цельные цепочки."""
if name == "write_slow":
return class_share(
classes,
variant,
name,
"server_slow_traces",
"server_slow",
)
if name == "upload_bad":
return class_share(classes, variant, name, "chains")
return class_share(classes, variant, name)
def log_share(
classes: list[Mapping[str, Any]],
variant: str,
name: str,
*,
server_slow: bool = False,
) -> tuple[int, int] | None:
"""Записи журнала класса, у которых `trace_id` ведёт на трассу в Tempo,
из записей с `trace_id`; `server_slow` — у медленных записей только
медленные по серверу (их правило выбора обязано сохранить)."""
if server_slow and name == "write_slow":
return class_share(
classes,
variant,
name,
"server_slow_linked",
"server_slow",
)
return class_share(
classes,
variant,
name,
"log_to_trace",
"logs_with_trace",
)
def quantile_error(rows: list[Mapping[str, Any]]) -> dict[str, float | None]:
"""Медиана по повторам относительной ошибки p95 и p99 из buckets против
журнала, по маршрутам (ошибка считается в каждом прогоне, а не по
медианам квантилей)."""
out = {}
for route in ("read", "write"):
for q in ("p95", "p99"):
errors = []
for row in rows:
b = num(row.get(f"q_{route}_{q}_buckets"))
log = num(row.get(f"q_{route}_{q}_log"))
if b is not None and log:
errors.append(abs(b - log) / log)
out[f"{route}_{q}"] = statistics.median(errors) if errors else None
return out
def verdicts(
rows: list[Mapping[str, Any]],
classes: list[Mapping[str, Any]],
collector_memory_mib: float,
) -> list[Verdict]:
g = by_variant(rows)
out = []
cpu = "replica_cpu_ms_per_req"
def m(variant: str, key: str) -> float | None:
return med(g.get(variant, []), key)
def eff(variant: str, reference: str, key: str) -> bool | None:
return effect(g, variant, reference, key)
# H1
h = Verdict("H1")
full_cpu, off_cpu = m("full", cpu), m("off", cpu)
r = ratio(full_cpu, off_cpu)
h.need(
"CPU реплик на запрос ≥ +10 %",
None if r is None else bool(eff("full", "off", cpu)) and r >= 1.10,
f"{fmt(full_cpu, 3)} против {fmt(off_cpu, 3)} мс",
)
p50 = (m("full", "light_p50"), m("off", "light_p50"))
p99 = (m("full", "light_p99"), m("off", "light_p99"))
d50 = None if None in p50 else p50[0] - p50[1]
d99 = None if None in p99 else p99[0] - p99[1]
h.need(
"p50 лёгких не больше +1 мс",
None if d50 is None else d50 <= 1,
f"Δ {fmt(d50)} мс",
)
rel50, rel99 = ratio(*p50), ratio(*p99)
h.need(
"p99 лёгких ≥ +2 мс и растёт сильнее p50",
None
if d99 is None or rel50 is None or rel99 is None
else d99 >= 2 and rel99 > rel50,
f"Δ {fmt(d99)} мс, рост p99 ×{fmt(rel99)} против p50 ×{fmt(rel50)}",
)
out.append(h)
# H2
h = Verdict("H2")
head_cpu = m("head", cpu)
h.need(
"надбавка CPU head ≤ половины надбавки full",
None
if None in (head_cpu, off_cpu, full_cpu)
else head_cpu - off_cpu <= 0.5 * (full_cpu - off_cpu),
f"head {fmt(head_cpu, 3)}, off {fmt(off_cpu, 3)}, full "
f"{fmt(full_cpu, 3)} мс",
)
tr = ratio(m("head", "tempo_spans_per_s"), m("full", "tempo_spans_per_s"))
h.need(
"приём Tempo ≤ 15 % full",
None if tr is None else tr <= 0.15,
f"{fmt(None if tr is None else tr * 100, 1)} %",
)
head_ratio = 0.1
for name in HEAD_CLASSES:
share = class_share(classes, "head", name)
if share is None or not share[1]:
h.need(f"доля трасс {name} около 0,1", None)
continue
lo, hi = binom_interval(*share)
h.need(
f"доля трасс {name} около 0,1",
lo <= head_ratio <= hi,
f"{share[0]}/{share[1]}, интервал {lo:.3f}–{hi:.3f}",
)
for name in HEAD_CLASSES:
logs = class_share(classes, "head", name, "logs")
h.need(
f"записи журнала {name} — 100 %",
None if logs is None else logs[0] == logs[1],
"" if logs is None else f"{logs[0]}/{logs[1]}",
)
# счёт в метрике — с начала процессов: у регистраций ещё пользователи
# prepare.js, поэтому «не меньше клиента»; медленные записи server span
# и клиент меряют по-разному — не сравниваются
for name in ("broken", "register", "upload_bad"):
metric = class_share(classes, "head", name, "metric")
h.need(
f"счёт {name} в метрике — 100 %",
None if metric is None else metric[0] >= metric[1],
"" if metric is None else f"{metric[0]}/{metric[1]}",
)
out.append(h)
# H3
h = Verdict("H3")
tail_effect = eff("tail", "full", cpu)
h.need(
"CPU реплик на запрос в пределах шума от full",
None if tail_effect is None else not tail_effect,
f"tail {fmt(m('tail', cpu), 3)}, full {fmt(full_cpu, 3)} мс",
)
peak = (
m("tail", "otel_collector_peak_mib"),
m("full", "otel_collector_peak_mib"),
)
h.need(
"пик памяти Collector ≥ full + 15 МиБ",
None if None in peak else peak[0] >= peak[1] + 15,
f"{fmt(peak[0], 1)} против {fmt(peak[1], 1)} МиБ",
)
ccpu = (m("tail", "otel_collector_cpu"), m("full", "otel_collector_cpu"))
h.need(
"CPU Collector не выше full больше чем на 20 %",
None if None in ccpu else ccpu[0] <= ccpu[1] * 1.2,
f"{fmt(ccpu[0], 3)} против {fmt(ccpu[1], 3)} ядра",
)
tr = ratio(m("tail", "tempo_spans_per_s"), m("full", "tempo_spans_per_s"))
h.need(
"приём Tempo ≤ 30 % full",
None if tr is None else tr <= 0.30,
f"{fmt(None if tr is None else tr * 100, 1)} %",
)
keep_all(h, classes, "tail", require_chain_bad=True)
chains = class_share(classes, "tail", "upload", "chains")
h.need(
"цепочки обычных загрузок ≥ 90 %",
None
if chains is None or not chains[1]
else chains[0] / chains[1] >= 0.9,
"" if chains is None else f"{chains[0]}/{chains[1]}",
)
refused = sum(values(g.get("tail", []), "col_span_lost")) + sum(
values(g.get("tail", []), "tail_dropped_too_early"),
)
h.need(
"нет отказов memory_limiter и dropped_too_early",
None if not g.get("tail") else refused == 0,
f"{refused:g}",
)
out.append(h)
# H4
h = Verdict("H4")
sr = ratio(m("metrics", "prom_series"), m("full", "prom_series"))
h.need(
"рядов Prometheus ≤ 40 % full",
None if sr is None else sr <= 0.4,
f"{fmt(None if sr is None else sr * 100, 1)} %",
)
ar = ratio(
m("metrics", "prom_samples_per_s"),
m("full", "prom_samples_per_s"),
)
h.need(
"отсчётов в секунду ≤ 40 % full",
None if ar is None else ar <= 0.4,
f"{fmt(None if ar is None else ar * 100, 1)} %",
)
for key, label in (
(cpu, "CPU реплик"),
("otel_collector_cpu", "CPU Collector"),
):
e = eff("metrics", "full", key)
h.need(f"{label} в пределах шума", None if e is None else not e)
coarse = quantile_error(g.get("metrics", []))
fine = quantile_error(g.get("full", []))
pairs = [
(coarse[k], fine[k])
for k in coarse
if coarse[k] is not None and fine[k] is not None
]
h.need(
"ошибка квантиля coarse ≥ 2 × default хотя бы на одном маршруте",
None if not pairs else any(c >= 2 * max(f, 1e-9) for c, f in pairs),
", ".join(
f"{k} {fmt(coarse[k], 3)} / {fmt(fine[k], 3)}" for k in coarse
),
)
out.append(h)
# H5
h = Verdict("H5")
# full — все редкие классы; tail и balanced — классы, которые их правила
# сохраняют (медленные записи — медленные по серверу)
for variant, names in (
("full", HEAD_CLASSES),
("tail", MUST_KEEP),
("balanced", MUST_KEEP),
):
for name in names:
share = log_share(
classes,
variant,
name,
server_slow=variant != "full",
)
if share is None or not share[1]:
h.need(f"{variant}: журнал → трасса {name} — 100 %", None)
continue
h.need(
f"{variant}: журнал → трасса {name} — 100 %",
share[0] == share[1],
f"{share[0]}/{share[1]}",
)
for name in HEAD_CLASSES:
share = log_share(classes, "head", name)
if share is None or not share[1]:
h.need(f"head: журнал → трасса {name} около 0,1", None)
continue
lo, hi = binom_interval(*share)
h.need(
f"head: журнал → трасса {name} около 0,1",
lo <= head_ratio <= hi,
f"{share[0]}/{share[1]}, интервал {lo:.3f}–{hi:.3f}",
)
out.append(h)
# H6
h = Verdict("H6")
telemetry_rows = [r for r in rows if r["variant"] != "off"]
lost = sum(sum(values(telemetry_rows, key)) for key in LOSS_KEYS)
h.need(
"потерь SDK и Collector и dropped_too_early нет",
None if not telemetry_rows else lost == 0,
f"{lost:g}",
)
gaps = values(telemetry_rows, "spans_gap")
h.need(
"span SDK → Collector → Tempo сходятся с точностью 1 % (выборка "
"отдельно)",
None if not gaps else max(gaps) <= 0.01,
f"наибольшее расхождение {fmt(max(gaps) * 100 if gaps else None)} %",
)
out.append(h)
# B1–B5 и H7
bs = criteria(g, classes, collector_memory_mib)
h = Verdict("H7")
for b in bs:
h.need(
f"{b.name} — {b.result}",
None if b.result == "нет данных" else b.result == "подтверждено",
)
out.append(h)
# H8
h = Verdict("H8")
t, lk, pr = (
m("full", k) for k in ("tempo_cpu", "loki_cpu", "prometheus_cpu")
)
h.need(
"у full CPU Tempo больше Loki и Prometheus вместе",
None if None in (t, lk, pr) else t > lk + pr,
f"{fmt(t, 3)} против {fmt(lk, 3)} + {fmt(pr, 3)}",
)
dt, dl, dp = (m("full", f"disk_{k}_bytes") for k in DISKS[::-1])
h.need(
"рост Tempo на диске больше Loki и Prometheus вместе",
None if None in (dt, dl, dp) else dt > dl + dp,
f"{fmt(dt, 0)} против {fmt(dl, 0)} + {fmt(dp, 0)} байт",
)
for variant in ("head", "tail", "balanced"):
v = m(variant, "tempo_cpu")
h.need(
f"CPU Tempo у {variant} ≤ половины full",
None if None in (v, t) else v <= 0.5 * t,
f"{fmt(v, 3)} против {fmt(t, 3)}",
)
out.append(h)
return [*out, *bs]
def keep_all(
h: Verdict,
classes: list[Mapping[str, Any]],
variant: str,
*,
require_chain_bad: bool,
) -> None:
labels = {
"broken": "трассы 500 API — 100 %",
"write_slow": "трассы медленных по серверу записей — 100 %",
"upload_bad": "цепочки неудачных загрузок — 100 %",
}
for name in MUST_KEEP:
if name == "upload_bad" and not require_chain_bad:
continue
share = kept(classes, variant, name)
h.need(
labels[name],
None if share is None else share[0] == share[1],
"" if share is None else f"{share[0]}/{share[1]}",
)
# регистрация — редкий быстрый маршрут: сохраняется с долей остальных,
# в вердикт не входит, показывается числом
share = class_share(classes, variant, "register")
if share is not None:
h.notes.append(
f"регистрации (редкий быстрый маршрут, справочно) {share[0]}/"
f"{share[1]}",
)
def criteria(
g: Mapping[str, list[Mapping[str, Any]]],
classes: list[Mapping[str, Any]],
collector_memory_mib: float,
) -> list[Verdict]:
"""Критерии сбалансированной конфигурации B1–B5 (спек `load-benchmark`,
«Цена телеметрии в отчёте»)."""
def m(variant: str, key: str) -> float | None:
return med(g.get(variant, []), key)
out = []
b = Verdict("B1")
keep_all(b, classes, "balanced", require_chain_bad=True)
out.append(b)
b = Verdict("B2")
tb, tf, tt = (
m(v, "tempo_spans_per_s") for v in ("balanced", "full", "tail")
)
b.need(
"приём Tempo ≤ 25 % full",
None if None in (tb, tf) or not tf else tb / tf <= 0.25,
f"{fmt(ratio(tb, tf) and ratio(tb, tf) * 100, 1)} %",
)
b.need(
"приём Tempo меньше, чем у tail",
None if None in (tb, tt) else tb < tt,
f"{fmt(tb, 1)} против {fmt(tt, 1)} span/с",
)
sr = ratio(m("balanced", "prom_series"), m("full", "prom_series"))
b.need(
"рядов Prometheus ≤ 40 % full",
None if sr is None else sr <= 0.4,
f"{fmt(None if sr is None else sr * 100, 1)} %",
)
out.append(b)
b = Verdict("B3")
rows = g.get("balanced", [])
lost = sum(sum(values(rows, key)) for key in LOSS_KEYS)
b.need(
"потерь и отброшенных до решения нет",
None if not rows else lost == 0,
f"{lost:g}",
)
peaks = values(rows, "otel_collector_peak_mib")
b.need(
"пик памяти Collector ≤ 2/3 лимита",
None if not peaks else max(peaks) <= collector_memory_mib * 2 / 3,
f"{fmt(max(peaks) if peaks else None, 1)} из "
f"{collector_memory_mib:g} МиБ",
)
out.append(b)
b = Verdict("B4")
cb, co = (
m("balanced", "replica_cpu_ms_per_req"),
m(
"off",
"replica_cpu_ms_per_req",
),
)
b.need(
"CPU реплик на запрос не больше +25 %",
None if None in (cb, co) or not co else cb / co <= 1.25,
f"{fmt(cb, 3)} против {fmt(co, 3)} мс",
)
for key, limit in (("light_p95", 3), ("light_p99", 10)):
a, z = m("balanced", key), m("off", key)
b.need(
f"{key} не больше +{limit} мс",
None if None in (a, z) else a - z <= limit,
f"Δ {fmt(None if None in (a, z) else a - z)} мс",
)
out.append(b)
b = Verdict("B5")
for name in MUST_KEEP:
share = log_share(classes, "balanced", name, server_slow=True)
b.need(
f"журнал → трасса {name}",
None if share is None else share[0] == share[1],
"" if share is None else f"{share[0]}/{share[1]}",
)
out.append(b)
return out
# ------------------------------------------------------------- сводка
ORDER = ("off", "full", "full-aa", "head", "tail", "metrics", "balanced")
def spread(rows: list[Mapping[str, Any]], key: str, digits: int = 2) -> str:
return reconcile.spread((r.get(key) for r in rows), digits)
def render_summary(
meta: Mapping[str, Any],
rows: list[Mapping[str, Any]],
classes: list[Mapping[str, Any]],
comps: list[Mapping[str, Any]],
) -> str:
g = by_variant(rows)
names = [n for n in ORDER if n in g]
profile = meta.get("profile", {})
info = meta.get("telemetry_cost", {})
collector_mib = (
reconcile_size(profile.get("collector_memory", "256m")) / 2**20
)
lines = [
f"# Цена телеметрии — метка {meta.get('label')}",
"",
f"Commit `{meta.get('git', {}).get('commit')}`, dirty "
f"{meta.get('git', {}).get('dirty')}; профиль {profile.get('name')}; "
f"окна {info.get('windows')} с; повторов {profile.get('repeats')}; "
f"статус {meta.get('status')}.",
"",
"Медиана и разброс (min–max) повторов; шум — |full − full-aa|.",
"",
"## Задержки лёгких запросов (чистое окно), мс",
"",
"| Вариант | p50 | p95 | p99 | ошибок |",
"|---|---|---|---|---|",
]
for name in names:
r = g[name]
lines.append(
f"| {name} | {spread(r, 'light_p50')} | {spread(r, 'light_p95')} "
f"| {spread(r, 'light_p99')} | {spread(r, 'light_errors', 0)} |",
)
lines += [
"",
"## Остальные классы (чистое окно): p95, мс",
"",
"| Вариант | " + " | ".join(CLASSES) + " |",
"|---|" + "---|" * len(CLASSES),
]
for name in names:
lines.append(
f"| {name} | "
+ " | ".join(spread(g[name], f"{c}_p95", 1) for c in CLASSES)
+ " |",
)
lines += [
"",
"## CPU и память",
"",
"| Вариант | CPU реплик на запрос, мс | реплики, ядра | приложение "
"| конвейер телеметрии | прочее | Collector пик, МиБ |",
"|---|---|---|---|---|---|---|",
]
for name in names:
r = g[name]
lines.append(
f"| {name} | {spread(r, 'replica_cpu_ms_per_req', 3)} | "
f"{spread(r, 'replica_cpu_cores', 3)} | "
f"{spread(r, 'app_cpu_cores', 3)} | "
f"{spread(r, 'telemetry_cpu_cores', 3)} | "
f"{spread(r, 'rest_cpu_cores', 3)} | "
f"{spread(r, 'otel_collector_peak_mib', 1)} |",
)
noise_cpu = noise(g, "replica_cpu_ms_per_req")
lines += [
"",
f"Шум CPU реплик на запрос (A/A): {noise_cpu:.4f} мс.",
"",
"### По компонентам: CPU, ядра (медиана), пик памяти, МиБ",
"",
"| Вариант | "
+ " | ".join(f"{role} CPU / пик" for role in TELEMETRY_ROLES)
+ " |",
"|---|" + "---|" * len(TELEMETRY_ROLES),
]
for name in names:
r = g[name]
cells = []
for role in TELEMETRY_ROLES:
p = role.replace("-", "_")
cells.append(
f"{fmt(med(r, f'{p}_cpu'), 3)} / "
f"{fmt(med(r, f'{p}_peak_mib'), 1)}",
)
lines.append(f"| {name} | " + " | ".join(cells) + " |")
lines += [
"",
"## Приём по сигналам (чистое окно, в секунду) и рост на диске",
"",
"| Вариант | span Collector | записей | точек | span Tempo | байт Tempo "
"| строк Loki | байт Loki | отсчётов Prometheus | рядов | диск "
"Prometheus / Loki / Tempo, МиБ |",
"|---|---|---|---|---|---|---|---|---|---|---|",
]
for name in names:
r = g[name]
disk = " / ".join(
fmt(
None
if med(r, f"disk_{d}_bytes") is None
else med(r, f"disk_{d}_bytes") / 2**20,
1,
)
for d in DISKS
)
lines.append(
f"| {name} | {spread(r, 'col_accepted_spans_per_s', 0)} | "
f"{spread(r, 'col_accepted_logs_per_s', 0)} | "
f"{spread(r, 'col_accepted_points_per_s', 0)} | "
f"{spread(r, 'tempo_spans_per_s', 0)} | "
f"{spread(r, 'tempo_bytes_per_s', 0)} | "
f"{spread(r, 'loki_lines_per_s', 0)} | "
f"{spread(r, 'loki_bytes_per_s', 0)} | "
f"{spread(r, 'prom_samples_per_s', 0)} | "
f"{spread(r, 'prom_series', 0)} | {disk} |",
)
run_s = sum(info.get("windows") or [0])
lines += [
"",
"Пересчёт на сутки — приём в секунду × 86 400; срок хранения 7 суток "
"— × 604 800 (рост на диске за прогон — журнал предзаписи, без "
f"сжатых блоков: прогон {run_s} с).",
"",
"## Потери по звеньям и отброшенное выборкой (весь прогон)",
"",
"| Вариант | span SDK выбрано / отброшено в голове | SDK отправил | "
"Collector принял / отправил | решения tail: сохр. / нет | Tempo "
"принял | потери SDK span / записей | потери Collector span / "
"записей | dropped_too_early | расхождение |",
"|---|---|---|---|---|---|---|---|---|---|",
]
for name in names:
r = g[name]
lines.append(
f"| {name} | {spread(r, 'sdk_spans_sampled', 0)} / "
f"{spread(r, 'sdk_spans_dropped', 0)} | "
f"{spread(r, 'sdk_spans_exported', 0)} | "
f"{spread(r, 'col_spans_accepted', 0)} / "
f"{spread(r, 'col_spans_sent', 0)} | "
f"{spread(r, 'tail_sampled', 0)} / "
f"{spread(r, 'tail_not_sampled', 0)} | "
f"{spread(r, 'tempo_spans_total', 0)} | "
f"{spread(r, 'sdk_span_lost', 0)} / {spread(r, 'sdk_log_lost', 0)} "
f"| {spread(r, 'col_span_lost', 0)} / "
f"{spread(r, 'col_log_lost', 0)} | "
f"{spread(r, 'tail_dropped_too_early', 0)} | "
f"{spread(r, 'spans_gap', 4)} |",
)
lines += [
"",
"## Время до ready и итоги заданий",
"",
"| Вариант | p50, с | p95, с | принято / ready | bad принято / "
"failed | медленных записей | недосланных |",
"|---|---|---|---|---|---|---|",
]
for name in names:
r = g[name]
lines.append(
f"| {name} | {spread(r, 'ttr_p50_s', 3)} | "
f"{spread(r, 'ttr_p95_s', 3)} | "
f"{int(sum(values(r, 'accepted')))} / "
f"{int(sum(values(r, 'ready')))} | "
f"{int(sum(values(r, 'bad_accepted')))} / "
f"{int(sum(values(r, 'bad_failed')))} | "
f"{int(sum(values(r, 'slow_writes')))} | "
f"{int(sum(values(r, 'dropped')))} |",
)
lost_runs = [r["run"] for r in rows if "lost" in (r.get("flags") or "")]
if lost_runs:
lines += ["", f"Принятая работа не дошла до итога: {lost_runs}."]
lines += [
"",
"## Что сохраняет каждый вариант (сумма повторов)",
"",
"Трасса по `X-Request-Id`, цепочка загрузки (server span API, "
"producer relay, consumer worker с итогом), запись журнала, переход "
"журнал → трасса, счёт в метрике (конец прогона).",
"",
"| Вариант | Класс | по клиенту | трасс | цепочек | записей | с "
"trace_id | журнал → трасса | метрика | медленных по серверу / их "
"трасс |",
"|---|---|---|---|---|---|---|---|---|---|",
]
for name in names:
for cls in RARE:
rows_c = [
r
for r in classes
if r["variant"] == name and r["class"] == cls
]
if not rows_c:
continue
def total(
key: str,
items: list[Mapping[str, Any]] = rows_c,
) -> str:
found = values(items, key)
return "—" if not found else f"{int(sum(found))}"
lines.append(
f"| {name} | {cls} | {total('client')} | {total('traces')} | "
f"{total('chains')} | {total('logs')} | "
f"{total('logs_with_trace')} | {total('log_to_trace')} | "
f"{total('metric')} | {total('server_slow')} / "
f"{total('server_slow_traces')} |",
)
lines += [
"",
"## Точность квантилей: из buckets против журнала (чистое окно), с",
"",
"| Вариант | read p95 | read p99 | write p95 | write p99 |",
"|---|---|---|---|---|",
]
for name in names:
r = g[name]
cells = []
for route in ("read", "write"):
for q in ("p95", "p99"):
cells.append(
f"{fmt(med(r, f'q_{route}_{q}_buckets'), 4)} / "
f"{fmt(med(r, f'q_{route}_{q}_log'), 4)}",
)
lines.append(f"| {name} | " + " | ".join(cells) + " |")
lines += ["", "## Гипотезы и критерии", ""]
for verdict in verdicts(rows, classes, collector_mib):
lines += [verdict.render(), ""]
flagged = [f"{r['run']}: {r['flags']}" for r in rows if r.get("flags")]
lines += ["## Пометки прогонов", ""]
lines += [f"- {item}" for item in flagged] or ["нет"]
warnings = meta.get("host_warnings") or []
if warnings:
lines += ["", "## Предупреждения о хосте", ""]
lines += [f"- {w}" for w in warnings]
lines += [
"",
"Графики: `cpu-by-group.svg`, `latency-light.svg`, `ingest.svg`, "
"`traces-by-class.svg`.",
"",
]
del comps
return "\n".join(lines)
def reconcile_size(text: str) -> int:
units = {"k": 2**10, "m": 2**20, "g": 2**30}
text = str(text).strip().lower()
if text and text[-1] in units:
return int(float(text[:-1]) * units[text[-1]])
return int(float(text))
def grouped_bars(
title: str,
unit: str,
groups: list[str],
series: list[tuple[str, list[float | None]]],
*,
stacked: bool = False,
) -> str:
"""Столбики по вариантам: рядом (или стопкой) несколько рядов."""
width, height = 860, 420
left, right, top, bottom = 70, 20, 60, 70
svg = plots.Svg(width, height, title)
totals = [
sum(v or 0 for _, vals in series for v in [vals[i]])
if stacked
else max((vals[i] or 0) for _, vals in series)
for i in range(len(groups))
]
high = max(totals + [0.0]) or 1.0
ticks = plots.nice_ticks(high)
top_value = ticks[-1]
plot_h = height - top - bottom
plot_w = width - left - right
def y(value: float) -> float:
return top + plot_h * (1 - value / top_value)
for tick in ticks:
svg.line(left, y(tick), width - right, y(tick))
svg.text(left - 6, y(tick) + 4, f"{tick:g}", anchor="end", size=11)
svg.text(18, top + plot_h / 2, unit, rotate=True, anchor="middle", size=11)
slot = plot_w / max(1, len(groups))
bars = 1 if stacked else max(1, len(series))
bar_w = min(40.0, slot * 0.7 / bars)
for i, group in enumerate(groups):
x0 = left + slot * i + (slot - bar_w * bars) / 2
base = 0.0
for j, (label, vals) in enumerate(series):
value = vals[i]
if value is None:
continue
color = plots.SERIES[j % len(plots.SERIES)]
if stacked:
svg.bar(
x0,
y(base + value),
bar_w,
y(base) - y(base + value),
color,
f"{group} {label}: {value:.3g}",
top=j == len(series) - 1,
)
base += value
else:
x = x0 + j * bar_w
svg.bar(
x,
y(value),
bar_w - 2,
y(0) - y(value),
color,
f"{group} {label}: {value:.3g}",
top=True,
)
svg.text(
left + slot * i + slot / 2,
height - bottom + 18,
group,
anchor="middle",
size=11,
)
svg.legend(
[
(plots.SERIES[j % len(plots.SERIES)], label, "box")
for j, (label, _) in enumerate(series)
],
left,
height - 24,
plot_w,
)
return svg.render()
def render_plots(
rows: list[Mapping[str, Any]],
classes: list[Mapping[str, Any]],
) -> dict[str, str]:
g = by_variant(rows)
names = [n for n in ORDER if n in g]
def series(keys: list[tuple[str, str]]) -> list[tuple[str, list[Any]]]:
return [
(label, [med(g[n], key) for n in names]) for key, label in keys
]
out = {
"cpu-by-group.svg": grouped_bars(
"CPU по группам в чистом окне",
"ядра",
names,
series(
[
("app_cpu_cores", "приложение"),
("telemetry_cpu_cores", "конвейер телеметрии"),
("rest_cpu_cores", "прочее"),
],
),
stacked=True,
),
"latency-light.svg": grouped_bars(
"Лёгкие запросы: p50, p95, p99",
"мс",
names,
series(
[
("light_p50", "p50"),
("light_p95", "p95"),
("light_p99", "p99"),
],
),
),
}
full = g.get("full", [])
def relative(key: str) -> list[float | None]:
base = med(full, key)
return [
None
if med(g[n], key) is None or not base
else med(g[n], key) / base * 100
for n in names
]
out["ingest.svg"] = grouped_bars(
"Приём по сигналам, % от full",
"%",
names,
[
("span Tempo", relative("tempo_spans_per_s")),
("строк Loki", relative("loki_lines_per_s")),
("отсчётов Prometheus", relative("prom_samples_per_s")),
],
)
shares = []
for cls in RARE:
vals = []
for n in names:
share = class_share(classes, n, cls)
vals.append(
None
if share is None or not share[1]
else share[0] / share[1] * 100,
)
shares.append((cls, vals))
out["traces-by-class.svg"] = grouped_bars(
"Сохранённые трассы редких событий, %",
"%",
names,
shares,
)
return out
def read_csv(path: Path) -> list[dict[str, Any]]:
return overload.read_csv(path)
# ------------------------------------------------------------ selftest
def selftest() -> None:
table = variants(0.1, 30)
assert set(table) == set(ORDER)
assert sdk_env(table["off"]) == {}
assert sdk_env(table["head"])["TELEMETRY__TRACE_SAMPLE_RATIO"] == "0.1"
assert sdk_env(table["metrics"])["TELEMETRY__METRIC_BUCKETS"] == "coarse"
assert wait_after(table["full"]) == 35
assert wait_after(table["metrics"]) == 50
assert wait_after(table["tail"]) == 52
assert wait_after(table["balanced"]) == 35
assert wait_after(table["off"]) == 0
compose = json.loads(variant_compose(table["balanced"], ROOT))
assert compose["services"]["otel-collector"]["command"][1].endswith(
"layer.yaml",
)
assert compose["services"]["prometheus"]["volumes"][0]["target"] == (
PROMETHEUS_TARGET
)
assert "services" in json.loads(variant_compose(table["off"], ROOT))
plan = plan_runs(ORDER, 3, 20261114)
assert len(plan) == 21
for a, b in zip(plan, plan[1:], strict=False):
assert a["variant"] != b["variant"], (a, b)
assert {p["variant"] for p in plan[:7]} == set(ORDER)
base = (ROOT / PROMETHEUS_CONFIG).read_text(encoding="utf-8")
lean = (ROOT / PROMETHEUS_LEAN).read_text(encoding="utf-8")
assert lean_problems(lean, base) == [], lean_problems(lean, base)
assert lean_problems(lean, base.replace("15s", "20s", 1))
assert lean_problems(lean.replace(LEAN_END, "", 1), base)
base_cfg = {
"processors": {"memory_limiter": {}, "batch": {}},
"service": {
"pipelines": {
"traces": {"processors": ["memory_limiter", "batch"]},
"logs": {"processors": ["memory_limiter", "batch"]},
},
},
}
good = json.loads(json.dumps(base_cfg))
good["processors"]["tail_sampling"] = {"policies": []}
good["service"]["pipelines"]["traces"]["processors"] = [
"memory_limiter",
"tail_sampling",
"batch",
]
assert collector_layer_problems(base_cfg, good) == []
bad = json.loads(json.dumps(good))
bad["service"]["pipelines"]["logs"]["processors"].append("extra")
assert any("logs" in p for p in collector_layer_problems(base_cfg, bad))
assert collector_layer_problems(base_cfg, base_cfg)
records = {
"api:a": [
{
"event": "Started API",
"telemetry_enabled": True,
"telemetry_trace_sample_ratio": 0.1,
"telemetry_metric_buckets": "default",
"telemetry_metric_export_interval_seconds": 15,
},
],
"worker": [{"event": "Started worker", "telemetry_enabled": False}],
}
found = startup_problems(records, table["head"])
assert found and all(p.startswith("worker:") for p in found), found
assert startup_problems({"relay": []}, table["off"])
assert (
layer_problems(
{"tail_sampling": True, "lean_prometheus": True},
table["balanced"],
)
== []
)
assert layer_problems({"tail_sampling": True}, table["full"])
t0_ms = 1_800_000_000_000
lines = [f"t-t0,{t0_ms}"]
reqs_spec = [
("read", 35_000, 5.0, 200),
("read", 36_000, 7.0, 200),
("write", 37_000, 9.0, 201),
("write", 125_000, 400.0, 201),
("broken", 50_000, 4.0, 500),
("register", 60_000, 500.0, 201),
("upload", 40_000, 80.0, 202),
("upload_bad", 41_000, 70.0, 202),
("me", 45_000, 3.0, 200),
("me", 46_000, 3.0, 401),
]
for n, (cls, offset, dur, status) in enumerate(reqs_spec):
aid = f"a{n}" if cls.startswith("upload") else "-"
lines.append(
f"t,{cls},{t0_ms + offset},{dur},{status},r{n:031x},inst{n % 2},"
f"{aid}",
)
lines.append("garbage")
t0, reqs = parse_k6("\n".join(lines))
assert t0 == t0_ms / 1000 and len(reqs) == len(reqs_spec)
bounds = window_bounds(t0, (30, 90, 30))
clean = inside(reqs, bounds["clean"])
assert len(clean) == 9, len(clean)
light = latency(clean, LIGHT)
assert light["n"] == 5 and light["errors"] == 1 and light["p99"] == 9.0
ids = rare_ids(reqs, 250)
assert len(ids["write_slow"]) == 1 and len(ids["broken"]) == 1
lo, hi = binom_interval(3, 30)
assert lo < 0.1 < hi, (lo, hi)
lo, hi = binom_interval(30, 30)
assert hi == 1.0 and lo > 0.88
lo, hi = binom_interval(0, 30)
assert lo == 0.0 and 0.1 < hi < 0.13, hi
assert parse_du("100\t/v/prometheus\n200 /v/tempo\nx") == {
"prometheus": 100,
"tempo": 200,
}
ns = int(t0 * 1e9)
sampler = []
for second in range(0, 160, 1):
t = ns + second * 10**9
sampler.append(
f"cg {t} - aaaaaaaaaaaa {second * 400_000} {second * 10} 0 0 "
f"{100 * 2**20} 0",
)
sampler.append(
f"cg {t} - cccccccccccc {second * 100_000} {second * 10} "
f"{second} 0 {50 * 2**20} 0",
)
sampler.append(f"vm {t} 10 20 0")
sampler.append(f"mp {t} cccccccccccc {80 * 2**20} {60 * 2**20} 0")
meta = overload.Meta(200, "ready", "-", 3, t0 + 41)
metas = {
"a6": meta,
"a7": overload.Meta(200, "failed", "invalid_image", 0, t0 + 42),
}
lookup = {
"classes": {
"broken": {
"traces": 1,
"logs": 1,
"logs_with_trace": 1,
"log_to_trace": 1,
},
},
"totals": {
"sdk_span_exported_ok": 1000,
"col_accepted_spans": 1000,
"col_sent_spans": 1000,
"tempo_spans": 995,
"sdk_span_processed_lost": 0,
},
"clean": {
"tempo_spans": {"delta": 900, "seconds": 90},
"prom_series": {"last": 5000},
},
"rare_metrics": {"broken": 1},
"quantiles": {"read": {"p95_buckets": 0.01, "p95_log": 0.007}},
"layers": {"tail_sampling": False, "lean_prometheus": False},
}
src = Sources(
run={"run": "full-r1", "variant": "full", "repeat": 1, "order": 1},
variant=table["full"],
windows=(30, 90, 30),
t0=t0,
reqs=reqs,
dropped=0,
sampler_text="\n".join(sampler),
containers={
"aaaaaaaaaaaa": {"name": "mtusi-bench-api-1", "role": "api"},
"cccccccccccc": {
"name": "mtusi-bench-otel-collector",
"role": "otel-collector",
},
},
metas=metas,
lookup=lookup,
du={"prometheus": 1, "loki": 2, "tempo": 3},
slow_ms=250,
)
comps = components(src)
assert len(comps) == 2
api = next(c for c in comps if c["role"] == "api")
assert abs(api["cpu_cores"] - 0.4) < 1e-6, api
col = next(c for c in comps if c["role"] == "otel-collector")
assert col["peak_mib"] == 80.0 and col["throttle"] == 0.1, col
row = run_row(src, comps)
assert set(row) == set(RUN_FIELDS), set(RUN_FIELDS) ^ set(row)
assert row["tempo_spans_per_s"] == 10.0
assert row["ready"] == 1 and row["bad_failed"] == 1
assert row["spans_gap"] == 0.005
assert "lost" not in row["flags"]
assert row["q_read_p95_buckets"] == 0.01
assert row["replica_cpu_ms_per_req"] is not None
crow = class_rows(src)
assert {r["class"] for r in crow} == set(RARE)
events = events_for_lookup(
reqs,
t0,
(30, 90, 30),
slow_ms=250,
sample_me=10,
seed=1,
collected_at=t0 + 300,
)
assert events["classes"]["write_slow"] and len(events["sample_me"]) == 1
rows = []
classes = []
rng = random.Random(7)
for variant in ORDER:
for repeat in (1, 2, 3):
base_cpu = {"off": 1.0, "head": 1.05}.get(variant, 1.2)
rows.append(
{
"run": f"{variant}-r{repeat}",
"variant": variant,
"repeat": repeat,
"replica_cpu_ms_per_req": base_cpu + rng.random() * 0.01,
"light_p50": 3.0,
"light_p95": 6.0,
"light_p99": 9.0 if variant == "off" else 12.0,
"tempo_spans_per_s": {
"head": 10,
"tail": 20,
"balanced": 15,
}.get(variant, 100),
"prom_series": 1500
if variant in ("metrics", "balanced")
else 5000,
"prom_samples_per_s": 30
if variant in ("metrics", "balanced")
else 100,
"otel_collector_peak_mib": 90
if variant in ("tail", "balanced")
else 60,
"otel_collector_cpu": 0.1,
"tempo_cpu": 0.2
if variant in ("full", "full-aa", "metrics")
else 0.05,
"loki_cpu": 0.05,
"prometheus_cpu": 0.05,
"disk_tempo_bytes": 3e6,
"disk_loki_bytes": 1e6,
"disk_prometheus_bytes": 1e6,
"spans_gap": 0.001,
"q_read_p95_buckets": 0.02
if variant == "metrics"
else 0.0075,
"q_read_p95_log": 0.007,
**dict.fromkeys(LOSS_KEYS, 0),
"flags": "",
},
)
for cls in RARE:
client = 30
keep = 3 if variant == "head" else client
classes.append(
{
"variant": variant,
"class": cls,
"client": client,
"traces": keep,
"chains": keep,
"logs": client,
"logs_with_trace": client,
"log_to_trace": keep,
"metric": client,
"server_slow": client,
"server_slow_traces": keep,
"server_slow_linked": keep,
},
)
result = {v.name: v.result for v in verdicts(rows, classes, 256)}
assert result["B1"] == "подтверждено", result
assert result["B3"] == "подтверждено", result
assert result["H2"] == "подтверждено", [
v.render() for v in verdicts(rows, classes, 256) if v.name == "H2"
]
assert result["H4"] == "подтверждено", [
v.render() for v in verdicts(rows, classes, 256) if v.name == "H4"
]
assert result["H8"] == "подтверждено", result
text = render_summary(
{
"label": "self",
"profile": {"name": "telemetry", "repeats": 3},
"telemetry_cost": {"windows": [30, 90, 30]},
},
rows,
classes,
[],
)
assert "## Гипотезы и критерии" in text and "**B5:" in text
for name, svg in render_plots(rows, classes).items():
assert svg.startswith("<svg") and name.endswith(".svg")
print("telemetry_cost selftest: ok")
if __name__ == "__main__":
if sys.argv[1:] == ["--selftest"]:
selftest()
sys.exit(0)
print(__doc__)
sys.exit(2)