"""Профиль цены телеметрии без docker: варианты, план, разбор, вердикты. Design change `track1-telemetry-cost`, решения 2, 6, 7, 9–12; оркестратор — `bench/run.py` (профили `telemetry`, `telemetry_quick`). Только stdlib, `src` не импортирует. Здесь — таблица вариантов, план кругами, разбор строк k6 (`k6/telemetry.js`), сэмплера `cost`, обхода метаданных, выборки (`telemetry_lookup.py`) и размера volume; окна; CPU на запрос, приём, рост на диске, потери по звеньям, доли классов редких событий, точность квантилей; строки `runs.csv`, `components.csv`, `classes.csv`; правило шума, точный биномиальный интервал, вердикты H1–H8 и B1–B5, пометки; `summary.md` и четыре SVG; проверки конфигураций до стенда (слой Collector, копия Prometheus) и записей о старте процессов. uv run python bench/telemetry_cost.py --selftest """ import json import math import random import statistics import sys from collections.abc import Iterable, Mapping from dataclasses import dataclass from pathlib import Path from typing import Any import overload import plots import reconcile import scaling ROOT = Path(__file__).resolve().parents[1] COLLECTOR_CONFIG = "infra/otel/collector.yaml" PROMETHEUS_CONFIG = "infra/otel/prometheus/prometheus.yml" TAIL_TYPICAL = "bench/otel/tail-typical.yaml" TAIL_BALANCED = "bench/otel/tail-balanced.yaml" PROMETHEUS_LEAN = "bench/otel/prometheus-lean.yml" LEAN_BEGIN = "# bench-lean: begin" LEAN_END = "# bench-lean: end" # пути конфигураций в контейнерах (docker-compose.yml) COLLECTOR_TARGET = "/etc/otelcol-contrib/config.yaml" LAYER_TARGET = "/etc/otelcol-contrib/layer.yaml" PROMETHEUS_TARGET = "/etc/prometheus/prometheus.yml" # сроки ожидания решения выбора трасс в Collector, с (decision_wait слоёв) DECISION_WAIT_S = {TAIL_TYPICAL: 30, TAIL_BALANCED: 10} SCRAPE_S = 15 EXPORT_MARGIN_S = 5 DECISION_MARGIN_S = 2 # классы запросов (решение 8); лёгкие — для цены LIGHT = ("read", "write", "me", "card") CLASSES = ( "read", "write", "me", "card", "download", "upload", "upload_bad", "broken", "register", ) OK_STATUS = { "read": 200, "write": 201, "me": 200, "card": 200, "download": 200, "upload": 202, "upload_bad": 202, "broken": 500, "register": 201, } # классы редких событий: что ищется в хранилищах RARE = ("broken", "write_slow", "register", "upload_bad", "upload") # редкие события, которые выбор в голове сохраняет с долей около 0,1 HEAD_CLASSES = ("broken", "write_slow", "register", "upload_bad") # сохранение, которое критерии tail и balanced требуют полным: ошибки, # медленные по серверу записи (≥ порога по `duration_ms` записи доступа — # той же величине, что server span) и цепочки неудачных загрузок. # Регистрация — редкий, но быстрый маршрут (~85 мс на стенде, а не 0,3–1 с, # как ждал design): у tail и balanced она сохраняется с долей остальных, это # и есть «что теряется» (design, «Уточнения apply») MUST_KEEP = ("broken", "write_slow", "upload_bad") ROLE_GROUPS = { "app": ("api", "worker", "relay"), "telemetry": ("otel-collector", "prometheus", "loki", "tempo"), } TELEMETRY_ROLES = ROLE_GROUPS["telemetry"] COMPONENT_FIELDS = [ "run", "variant", "repeat", "role", "name", "cpu_cores", "throttle", "mem_max_mib", "peak_mib", "anon_max_mib", ] CLASS_FIELDS = [ "run", "variant", "repeat", "class", "client", "traces", "chains", "logs", "logs_with_trace", "logs_sampled_flag", "log_to_trace", "metric", "server_slow", "server_slow_traces", "server_slow_linked", ] LATENCY_KEYS = ("n", "errors", "p50", "p95", "p99") INGEST = ( "col_accepted_spans", "col_accepted_logs", "col_accepted_points", "tempo_spans", "tempo_bytes", "loki_lines", "loki_bytes", "loki_sm_bytes", "prom_samples", ) DISKS = ("prometheus", "loki", "tempo") LOSS_KEYS = ( "sdk_span_lost", "sdk_log_lost", "col_span_lost", "col_log_lost", "col_point_refused", "tail_dropped_too_early", ) RUN_FIELDS = [ "run", "variant", "repeat", "order", "warm_s", "clean_s", "fault_s", *(f"{name}_{key}" for name in ("light", *CLASSES) for key in LATENCY_KEYS), "replica_cpu_ms_per_req", "replica_cpu_cores", "app_cpu_cores", "telemetry_cpu_cores", "rest_cpu_cores", *( f"{role.replace('-', '_')}_{key}" for role in TELEMETRY_ROLES for key in ("cpu", "throttle", "mem_max_mib", "peak_mib") ), *(f"{name}_per_s" for name in INGEST), "prom_series", *(f"disk_{name}_bytes" for name in DISKS), "sdk_spans_sampled", "sdk_spans_dropped", "sdk_spans_exported", "col_spans_accepted", "col_spans_sent", "tempo_spans_total", "tail_sampled", "tail_not_sampled", *LOSS_KEYS, "spans_gap", "ttr_p50_s", "ttr_p95_s", "accepted", "ready", "bad_accepted", "bad_failed", "slow_writes", "dropped", *( f"q_{route}_{q}_{src}" for route in ("read", "write") for q in ("p95", "p99") for src in ("buckets", "log") ), "flags", ] # пометки, при которых код команды в конце метки ненулевой (решение 11) FATAL_FLAGS = ("lost",) # калибровка (решение 8): CPU реплики в чистом окне, предел Collector, # Tempo и worker, p95 времени до ready CAL_REPLICA = (0.30, 0.60) CAL_LIMITED = 0.70 CAL_TTR_P95_S = 5.0 Z_ALPHA = 0.05 # ------------------------------------------------------------- варианты def variants(head_ratio: float, metrics_interval: float) -> dict[str, Any]: """Таблица вариантов (решение 2): настройки SDK трёх процессов, слой Collector и конфигурация Prometheus. Loki и Tempo — без изменений.""" full = { "telemetry": True, "ratio": 1.0, "buckets": "default", "interval": 15, "collector_layer": None, "prometheus": None, } return { "off": {"telemetry": False}, "full": full, "full-aa": dict(full), "head": {**full, "ratio": head_ratio}, "tail": {**full, "collector_layer": TAIL_TYPICAL}, "metrics": { **full, "interval": metrics_interval, "buckets": "coarse", "prometheus": PROMETHEUS_LEAN, }, "balanced": { **full, "collector_layer": TAIL_BALANCED, "prometheus": PROMETHEUS_LEAN, }, } def sdk_env(variant: Mapping[str, Any]) -> dict[str, str]: """Окружение `TELEMETRY__*` реплик, worker и relay варианта (без включения и адреса — их даёт `telemetry.compose.yml`).""" if not variant.get("telemetry"): return {} return { "TELEMETRY__TRACE_SAMPLE_RATIO": f"{variant['ratio']:g}", "TELEMETRY__METRIC_BUCKETS": variant["buckets"], "TELEMETRY__METRIC_EXPORT_INTERVAL_SECONDS": f"{variant['interval']:g}", } def wait_after(variant: Mapping[str, Any]) -> float: """Ожидание отправок после нагрузки (решение 9): последняя отправка метрик + scrape и решение выбора трасс доходят до хранилищ раньше выборки.""" if not variant.get("telemetry"): return 0.0 metrics = variant["interval"] + SCRAPE_S + EXPORT_MARGIN_S layer = variant.get("collector_layer") # после решения span ещё уходят в Tempo, а счётчик Tempo доходит до # Prometheus через scrape (пробный tail без scrape — расхождение 8 %) decision = ( DECISION_WAIT_S[layer] + DECISION_MARGIN_S + SCRAPE_S + EXPORT_MARGIN_S if layer else 0 ) return float(max(metrics, decision)) def variant_compose(variant: Mapping[str, Any], root: Path) -> str: """`.work/variant.compose.yml` прогона (JSON — подмножество YAML): окружение SDK трёх процессов, у Collector — второй `--config` и bind слоя, у Prometheus — bind своей конфигурации на тот же путь (compose сливает volumes по пути в контейнере).""" services: dict[str, Any] = {} env = sdk_env(variant) for service in ("api", "worker", "relay"): if env: services[service] = {"environment": dict(env)} layer = variant.get("collector_layer") if layer: services["otel-collector"] = { "command": [ f"--config={COLLECTOR_TARGET}", f"--config={LAYER_TARGET}", ], "volumes": [bind(root / layer, LAYER_TARGET)], } lean = variant.get("prometheus") if lean: services["prometheus"] = { "volumes": [bind(root / lean, PROMETHEUS_TARGET)], } return json.dumps({"services": services}, indent=1) + "\n" def bind(source: Path, target: str) -> dict[str, Any]: return { "type": "bind", "source": str(source), "target": target, "read_only": True, "bind": {"create_host_path": False}, } def plan_runs( names: Iterable[str], repeats: int, seed: int, ) -> list[dict[str, Any]]: """Круги с перестановкой от зерна; круг не начинается с варианта, которым кончился прошлый: повтор одного варианта подряд не идёт.""" items = list(names) plan: list[dict[str, Any]] = [] previous = None for round_no in range(1, repeats + 1): order = list(items) random.Random(seed + round_no).shuffle(order) if len(order) > 1 and order[0] == previous: order[0], order[1] = order[1], order[0] for name in order: plan.append( { "run": f"{name}-r{round_no}", "variant": name, "repeat": round_no, "order": len(plan) + 1, }, ) previous = order[-1] if order else None return plan # ------------------------------------------------- проверки конфигураций def lean_problems(lean: str, base: str) -> list[str]: """Копия Prometheus без размеченных блоков совпадает с рабочей построчно (решение 7).""" kept: list[str] = [] inside = False for number, line in enumerate(lean.splitlines(), start=1): mark = line.strip() if mark == LEAN_BEGIN: if inside: return [f"line {number}: nested {LEAN_BEGIN}"] inside = True continue if mark == LEAN_END: if not inside: return [f"line {number}: {LEAN_END} without begin"] inside = False continue if not inside: kept.append(line) if inside: return [f"{LEAN_BEGIN} is not closed"] problems = [] base_lines = base.splitlines() for number, (a, b) in enumerate(zip(kept, base_lines, strict=False), 1): if a != b: problems.append( f"outside bench-lean blocks line {number} differs from " f"{PROMETHEUS_CONFIG}: {a!r} != {b!r}", ) break if not problems and len(kept) != len(base_lines): problems.append( f"outside bench-lean blocks {len(kept)} lines, " f"{PROMETHEUS_CONFIG} {len(base_lines)}", ) return problems def first_difference(a: Any, b: Any, path: str = "") -> str | None: if isinstance(a, dict) and isinstance(b, dict): for key in sorted(set(a) | set(b)): if key not in a or key not in b: return f"{path}.{key}".lstrip(".") found = first_difference(a[key], b[key], f"{path}.{key}") if found is not None: return found return None return None if a == b else (path.lstrip(".") or "") def collector_layer_problems( base: Mapping[str, Any], layered: Mapping[str, Any], ) -> list[str]: """Итоговая конфигурация со слоем отличается от рабочей только процессором `tail_sampling` и списком процессоров pipeline трасс — базовым с `tail_sampling` перед `batch` (решение 6).""" layered = json.loads(json.dumps(layered)) problems = [] processors = layered.get("processors") or {} if "tail_sampling" not in processors: problems.append("layer has no processors.tail_sampling") processors.pop("tail_sampling", None) traces = layered.get("service", {}).get("pipelines", {}).get("traces", {}) base_list = list(base["service"]["pipelines"]["traces"]["processors"]) if "batch" in base_list: expected = list(base_list) expected.insert(expected.index("batch"), "tail_sampling") else: expected = [*base_list, "tail_sampling"] if traces.get("processors") != expected: problems.append( f"traces processors {traces.get('processors')}, expected " f"{expected}", ) traces["processors"] = base_list diff = first_difference(dict(base), layered) if diff is not None: problems.append(f"layer changes {diff}") return problems STARTED = { "api": "Started API", "worker": "Started worker", "relay": "Started relay", } def startup_problems( records_by_process: Mapping[str, list[dict[str, Any]]], variant: Mapping[str, Any], ) -> list[str]: """Записи о старте реплик, worker и relay против варианта (решение 11): включена ли телеметрия, доля, набор границ, интервал.""" problems = [] for process, records in records_by_process.items(): kind = process.split(":", 1)[0] event = STARTED[kind] started = [r for r in records if r.get("event") == event] if not started: problems.append(f"{process}: no «{event}» record") continue record = started[-1] expected: dict[str, Any] = { "telemetry_enabled": bool(variant.get("telemetry")), } if variant.get("telemetry"): expected.update( { "telemetry_trace_sample_ratio": float(variant["ratio"]), "telemetry_metric_buckets": variant["buckets"], "telemetry_metric_export_interval_seconds": float( variant["interval"], ), }, ) for key, value in expected.items(): seen = record.get(key) if isinstance(value, float): ok = isinstance(seen, int | float) and math.isclose( float(seen), value, ) else: ok = seen == value if not ok: problems.append( f"{process}: {key} {seen!r}, expected {value!r}", ) return problems def layer_problems( layers: Mapping[str, bool], variant: Mapping[str, Any], ) -> list[str]: """Слои Collector и Prometheus по выборке (решение 9).""" problems = [] want_tail = bool(variant.get("collector_layer")) if bool(layers.get("tail_sampling")) != want_tail: problems.append( f"tail sampling series {layers.get('tail_sampling')}, " f"expected {want_tail}", ) want_lean = bool(variant.get("prometheus")) if bool(layers.get("lean_prometheus")) != want_lean: problems.append( f"lean Prometheus config {layers.get('lean_prometheus')}, " f"expected {want_lean}", ) return problems # ------------------------------------------------------------- разбор @dataclass(frozen=True) class Req: cls: str sent_ms: float duration_ms: float status: int request_id: str instance: str attachment_id: str @property def answered(self) -> float: return (self.sent_ms + self.duration_ms) / 1000 def parse_k6(text: str) -> tuple[float | None, list[Req]]: """Строки `t-t0,<мс>` и `t,<класс>,<отправка>,<длительность>,<статус>, ,,`.""" t0 = None out = [] for raw in text.splitlines(): parts = raw.strip().split(",") if parts[0] == "t-t0" and len(parts) == 2: t0 = float(parts[1]) / 1000 elif parts[0] == "t" and len(parts) == 8: out.append( Req( parts[1], float(parts[2]), float(parts[3]), int(parts[4]), parts[5], parts[6], parts[7], ), ) return t0, out def window_bounds( t0: float, windows: tuple[int, int, int], ) -> dict[str, tuple[float, float]]: warm, clean, fault = windows return { "warm": (t0, t0 + warm), "clean": (t0 + warm, t0 + warm + clean), "fault": (t0 + warm + clean, t0 + warm + clean + fault), "run": (t0, t0 + warm + clean + fault), } def inside(reqs: Iterable[Req], w: tuple[float, float]) -> list[Req]: return [r for r in reqs if w[0] * 1000 <= r.sent_ms < w[1] * 1000] def latency(reqs: list[Req], classes: Iterable[str]) -> dict[str, Any]: names = set(classes) chosen = [r for r in reqs if r.cls in names] ok = [r.duration_ms for r in chosen if r.status == OK_STATUS[r.cls]] return { "n": len(chosen), "errors": len(chosen) - len(ok), "p50": rounded(reconcile.percentile(ok, 50), 2), "p95": rounded(reconcile.percentile(ok, 95), 2), "p99": rounded(reconcile.percentile(ok, 99), 2), } def rounded(value: float | None, digits: int = 3) -> float | None: return None if value is None else round(value, digits) def rare_ids(reqs: list[Req], slow_ms: float) -> dict[str, list[str]]: """Id запросов классов редких событий (по клиенту): 500 вложения без объекта, медленные записи, регистрации, принятые загрузки `bad` и обычные.""" def ids(cls: str, status: int) -> list[str]: return [ r.request_id for r in reqs if r.cls == cls and r.status == status and r.request_id != "-" ] return { "broken": ids("broken", 500), "write_slow": [ r.request_id for r in reqs if r.cls == "write" and r.status == 201 and r.duration_ms >= slow_ms and r.request_id != "-" ], "register": ids("register", 201), "upload_bad": ids("upload_bad", 202), "upload": ids("upload", 202), } def parse_du(text: str) -> dict[str, int]: """`du -sb /v/<имя>` по строкам: имя volume → байт.""" out = {} for line in text.splitlines(): parts = line.split() if len(parts) == 2 and parts[0].isdigit(): out[parts[1].rstrip("/").rsplit("/", 1)[-1]] = int(parts[0]) return out @dataclass class Sources: """Всё снятое за прогон.""" run: Mapping[str, Any] variant: Mapping[str, Any] windows: tuple[int, int, int] t0: float reqs: list[Req] dropped: int sampler_text: str containers: Mapping[str, Mapping[str, str]] metas: Mapping[str, Any] lookup: Mapping[str, Any] | None du: Mapping[str, int] slow_ms: float not_drained: bool = False storage_down: bool = False oom: bool = False swapouts: int | None = None def components(src: Sources) -> list[dict[str, Any]]: """CPU, троттлинг и память каждого контейнера стенда в чистом окне.""" w0, w1 = window_bounds(src.t0, src.windows)["clean"] samples = scaling.parse_samples(src.sampler_text) mp = overload.parse_mp(src.sampler_text) rows = [] for (_, ident), points in sorted(samples.cg.items()): info = src.containers.get(ident) if not info: continue lives = overload.split_lives(points) cores = overload.role_cores([lives], w0, w1) in_window = [p for p in points if w0 * 1e9 <= p[0] <= w1 * 1e9] peaks = mp.get(ident, []) rows.append( { "run": src.run["run"], "variant": src.run["variant"], "repeat": src.run["repeat"], "role": info["role"], "name": info["name"], "cpu_cores": cores, "throttle": overload.throttle_share(lives, w0, w1), "mem_max_mib": ( round(max(p[5] for p in in_window) / 2**20, 1) if in_window else None ), "peak_mib": ( round(max(p[1] for p in peaks) / 2**20, 1) if peaks else None ), "anon_max_mib": ( round(max(p[2] for p in peaks) / 2**20, 1) if peaks else None ), }, ) return rows def group_cores(rows: list[dict[str, Any]]) -> dict[str, float | None]: out: dict[str, float] = {"app": 0.0, "telemetry": 0.0, "rest": 0.0} seen = False for row in rows: cores = row["cpu_cores"] if cores is None or row["role"] in ("k6", "sampler"): continue seen = True group = next( (g for g, roles in ROLE_GROUPS.items() if row["role"] in roles), "rest", ) out[group] += cores if not seen: return dict.fromkeys(out) return {key: round(value, 4) for key, value in out.items()} def num(value: Any) -> float | None: return reconcile.num(value) def zero(value: Any) -> float: return num(value) or 0.0 def losses(totals: Mapping[str, Any]) -> dict[str, float]: """Потери по звеньям (решение 9): SDK — отброшенное процессором и неудачные отправки; Collector — отказ приёма, ошибки и отказы отправки. Отброшенное выборкой (в голове и решения Collector) сюда не входит.""" return { "sdk_span_lost": zero(totals.get("sdk_span_processed_lost")) + zero(totals.get("sdk_span_export_failed")), "sdk_log_lost": zero(totals.get("sdk_log_processed_lost")) + zero(totals.get("sdk_log_export_failed")), "col_span_lost": sum( zero(totals.get(key)) for key in ( "col_refused_spans", "col_failed_spans", "col_send_failed_spans", "col_enqueue_failed_spans", ) ), "col_log_lost": sum( zero(totals.get(key)) for key in ( "col_refused_logs", "col_failed_logs", "col_send_failed_logs", "col_enqueue_failed_logs", ) ), "col_point_refused": zero(totals.get("col_refused_points")), "tail_dropped_too_early": zero(totals.get("tail_dropped_too_early")), } def spans_gap(totals: Mapping[str, Any]) -> float | None: """Доля, на которую span не сходятся по звеньям: SDK отправил → Collector принял, Collector отправил → Tempo принял (наибольшая из двух).""" pairs = ( ("sdk_span_exported_ok", "col_accepted_spans"), ("col_sent_spans", "tempo_spans"), ) gaps = [] for a, b in pairs: va, vb = num(totals.get(a)), num(totals.get(b)) if va is None or vb is None or not va: continue gaps.append(abs(va - vb) / va) return round(max(gaps), 4) if gaps else None def time_to_ready( reqs: list[Req], metas: Mapping[str, Any], ) -> tuple[float | None, float | None]: values = [] for r in reqs: meta = metas.get(r.attachment_id) if ( r.cls == "upload" and r.status == 202 and meta is not None and meta.status == "ready" and meta.updated_at is not None ): values.append(max(0.0, meta.updated_at - r.answered)) return ( rounded(reconcile.percentile(values, 50), 3), rounded(reconcile.percentile(values, 95), 3), ) def run_row( src: Sources, comps: list[dict[str, Any]], ) -> dict[str, Any]: bounds = window_bounds(src.t0, src.windows) clean = inside(src.reqs, bounds["clean"]) row: dict[str, Any] = { key: src.run[key] for key in ("run", "variant", "repeat", "order") } row.update( dict(zip(("warm_s", "clean_s", "fault_s"), src.windows, strict=True)), ) for name, classes in (("light", LIGHT), *((c, (c,)) for c in CLASSES)): for key, value in latency(clean, classes).items(): row[f"{name}_{key}"] = value replicas = [c for c in comps if c["role"] == "api"] replica_cores = ( sum(c["cpu_cores"] for c in replicas if c["cpu_cores"] is not None) if replicas else None ) answered = sum(1 for r in clean if r.instance != "-") clean_s = src.windows[1] row["replica_cpu_cores"] = rounded(replica_cores, 4) row["replica_cpu_ms_per_req"] = ( round(replica_cores * clean_s * 1000 / answered, 4) if replica_cores is not None and answered else None ) groups = group_cores(comps) for group in ("app", "telemetry", "rest"): row[f"{group}_cpu_cores"] = groups[group] for role in TELEMETRY_ROLES: found = [c for c in comps if c["role"] == role] prefix = role.replace("-", "_") item = found[0] if found else {} row[f"{prefix}_cpu"] = item.get("cpu_cores") row[f"{prefix}_throttle"] = item.get("throttle") row[f"{prefix}_mem_max_mib"] = item.get("mem_max_mib") row[f"{prefix}_peak_mib"] = item.get("peak_mib") lookup = src.lookup or {} clean_counts = lookup.get("clean", {}) for name in INGEST: item = clean_counts.get(name) or {} delta, seconds = item.get("delta"), item.get("seconds") row[f"{name}_per_s"] = ( round(delta / seconds, 3) if delta is not None and seconds else None ) row["prom_series"] = (clean_counts.get("prom_series") or {}).get("last") for name in DISKS: row[f"disk_{name}_bytes"] = src.du.get(name) totals = lookup.get("totals", {}) row["sdk_spans_sampled"] = totals.get("sdk_span_started_sampled") row["sdk_spans_dropped"] = totals.get("sdk_span_started_dropped") row["sdk_spans_exported"] = totals.get("sdk_span_exported_ok") row["col_spans_accepted"] = totals.get("col_accepted_spans") row["col_spans_sent"] = totals.get("col_sent_spans") row["tempo_spans_total"] = totals.get("tempo_spans") row["tail_sampled"] = totals.get("tail_sampled") row["tail_not_sampled"] = totals.get("tail_not_sampled") lost = losses(totals) if src.lookup else {} for key in LOSS_KEYS: row[key] = lost.get(key) row["spans_gap"] = spans_gap(totals) if src.lookup else None row["ttr_p50_s"], row["ttr_p95_s"] = time_to_ready(src.reqs, src.metas) accepted = [r for r in src.reqs if r.cls == "upload" and r.status == 202] bad = [r for r in src.reqs if r.cls == "upload_bad" and r.status == 202] def final(items: list[Req], status: str, code: str | None) -> int: count = 0 for r in items: meta = src.metas.get(r.attachment_id) if meta is None or meta.status != status: continue if code is not None and meta.error_code != code: continue count += 1 return count row["accepted"] = len(accepted) row["ready"] = final(accepted, "ready", None) row["bad_accepted"] = len(bad) row["bad_failed"] = final(bad, "failed", "invalid_image") row["slow_writes"] = len(rare_ids(src.reqs, src.slow_ms)["write_slow"]) row["dropped"] = src.dropped for route in ("read", "write"): values = (lookup.get("quantiles") or {}).get(route) or {} for q in ("p95", "p99"): for source in ("buckets", "log"): row[f"q_{route}_{q}_{source}"] = rounded( num(values.get(f"{q}_{source}")), 5, ) row["flags"] = " ".join(run_flags(src, row)) return row def run_flags(src: Sources, row: Mapping[str, Any]) -> list[str]: flags = [] if src.dropped: flags.append("dropped") if src.swapouts: flags.append("host_swap") if src.storage_down: flags.append("storage_down") if src.oom: flags.append("oom") if src.not_drained: flags.append("not_drained") if ( row["ready"] != row["accepted"] or row["bad_failed"] != row["bad_accepted"] ): flags.append("lost") if src.lookup and any(row.get(key) for key in LOSS_KEYS): flags.append("telemetry_loss") if src.run["variant"] == "full" and calibration_problems(row): flags.append("saturated") return flags def calibration_problems(row: Mapping[str, Any]) -> list[str]: """Границы калибровки решения 8 у варианта `full`.""" problems = [] cores = num(row.get("replica_cpu_cores")) if cores is not None: share = cores / 2 if not CAL_REPLICA[0] <= share <= CAL_REPLICA[1]: problems.append(f"replica CPU {share:.0%} of 1 CPU (30–60 %)") for key, limit_cpus in ( ("otel_collector_cpu", 0.5), ("tempo_cpu", 0.5), ): value = num(row.get(key)) if value is not None and value / limit_cpus >= CAL_LIMITED: problems.append(f"{key} {value / limit_cpus:.0%} of its limit") if row.get("dropped"): problems.append(f"dropped iterations {row['dropped']}") ttr = num(row.get("ttr_p95_s")) if ttr is not None and ttr >= CAL_TTR_P95_S: problems.append(f"time to ready p95 {ttr} s") return problems def class_rows(src: Sources) -> list[dict[str, Any]]: ids = rare_ids(src.reqs, src.slow_ms) lookup = src.lookup or {} found = lookup.get("classes", {}) metrics = lookup.get("rare_metrics", {}) rows = [] for name in RARE: item = found.get(name, {}) rows.append( { "run": src.run["run"], "variant": src.run["variant"], "repeat": src.run["repeat"], "class": name, "client": len(ids[name]), "traces": item.get("traces"), "chains": item.get("chains"), "logs": item.get("logs"), "logs_with_trace": item.get("logs_with_trace"), "logs_sampled_flag": item.get("logs_sampled_flag"), "log_to_trace": item.get("log_to_trace"), "metric": metrics.get(name), "server_slow": item.get("server_slow"), "server_slow_traces": item.get("server_slow_traces"), "server_slow_linked": item.get("server_slow_linked"), }, ) return rows def events_for_lookup( src_reqs: list[Req], t0: float, windows: tuple[int, int, int], *, slow_ms: float, sample_me: int, seed: int, collected_at: float, ) -> dict[str, Any]: """Вход `telemetry_lookup.py collect` (решение 9): id редких классов, выборка `GET /users/me` чистого окна, окна.""" bounds = window_bounds(t0, windows) me = [ r.request_id for r in inside(src_reqs, bounds["clean"]) if r.cls == "me" and r.status == 200 ] random.Random(seed).shuffle(me) return { "run": {"start": bounds["run"][0], "end": bounds["run"][1]}, "clean": {"start": bounds["clean"][0], "end": bounds["clean"][1]}, "search": {"start": t0 - 60, "end": collected_at}, "classes": rare_ids(src_reqs, slow_ms), "slow_ms": slow_ms, "sample_me": me[:sample_me], "routes": { "read": ["GET", "/api/v1/chats/{chat_id}/messages"], "write": ["POST", "/api/v1/chats/{chat_id}/messages"], }, } # ------------------------------------------------------ статистика def binom_cdf(k: int, n: int, p: float) -> float: if p <= 0: return 1.0 if p >= 1: return 1.0 if k >= n else 0.0 return sum( math.exp( math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1) + i * math.log(p) + (n - i) * math.log1p(-p), ) for i in range(k + 1) ) def binom_interval( k: int, n: int, alpha: float = Z_ALPHA, ) -> tuple[float, float]: """Точный (Клоппер — Пирсон) интервал доли k/n уровня 1 − alpha.""" if n <= 0: return (0.0, 1.0) def solve(f: Any) -> float: lo, hi = 0.0, 1.0 for _ in range(60): mid = (lo + hi) / 2 if f(mid): lo = mid else: hi = mid return (lo + hi) / 2 lower = ( 0.0 if k == 0 else solve(lambda p: 1 - binom_cdf(k - 1, n, p) < alpha / 2) ) upper = 1.0 if k == n else solve(lambda p: binom_cdf(k, n, p) > alpha / 2) return (lower, upper) def values(rows: Iterable[Mapping[str, Any]], key: str) -> list[float]: return [v for v in (num(r.get(key)) for r in rows) if v is not None] def med(rows: Iterable[Mapping[str, Any]], key: str) -> float | None: found = values(rows, key) return statistics.median(found) if found else None def by_variant( rows: Iterable[Mapping[str, Any]], ) -> dict[str, list[Mapping[str, Any]]]: out: dict[str, list[Mapping[str, Any]]] = {} for row in rows: out.setdefault(row["variant"], []).append(row) return out def noise(groups: Mapping[str, list[Mapping[str, Any]]], key: str) -> float: """Шум величины — |медиана full − медиана full-aa| (решение 12).""" a, b = ( med(groups.get("full", []), key), med(groups.get("full-aa", []), key), ) if a is None or b is None: return 0.0 return abs(a - b) def effect( groups: Mapping[str, list[Mapping[str, Any]]], variant: str, reference: str, key: str, ) -> bool | None: """Эффект: разбросы повторов не пересекаются и |Δ медианы| больше шума.""" a, b = ( values(groups.get(variant, []), key), values( groups.get(reference, []), key, ), ) if not a or not b: return None apart = max(a) < min(b) or max(b) < min(a) delta = abs(statistics.median(a) - statistics.median(b)) return apart and delta > noise(groups, key) class Verdict: """Условия гипотезы или критерия: «нет данных», если нечем проверить условие; «нет», если нарушено (причина названа).""" def __init__(self, name: str) -> None: self.name = name self.failed: list[str] = [] self.missing: list[str] = [] self.notes: list[str] = [] def need(self, label: str, ok: bool | None, detail: str = "") -> None: text = f"{label}{': ' + detail if detail else ''}" if ok is None: self.missing.append(text) elif ok: self.notes.append(text) else: self.failed.append(text) @property def result(self) -> str: if self.failed: return "нет" if self.missing: return "нет данных" return "подтверждено" def render(self) -> str: parts = [f"**{self.name}: {self.result}.**"] if self.failed: parts.append("Нарушено: " + "; ".join(self.failed) + ".") if self.missing: parts.append("Нет данных: " + "; ".join(self.missing) + ".") if self.notes: parts.append("Выполнено: " + "; ".join(self.notes) + ".") return " ".join(parts) def fmt(value: float | None, digits: int = 2) -> str: return "—" if value is None else f"{value:.{digits}f}" def ratio(a: float | None, b: float | None) -> float | None: return None if a is None or not b else a / b def class_share( classes: list[Mapping[str, Any]], variant: str, name: str, field: str = "traces", total_field: str = "client", ) -> tuple[int, int] | None: """Сумма найденного и всего (по умолчанию — счёт клиента) класса по повторам варианта.""" rows = [ r for r in classes if r["variant"] == variant and r["class"] == name ] if not rows or any( num(r.get(key)) is None for r in rows for key in (field, total_field) ): return None found = int(sum(zero(r.get(field)) for r in rows)) total = int(sum(zero(r.get(total_field)) for r in rows)) return found, total def kept( classes: list[Mapping[str, Any]], variant: str, name: str, ) -> tuple[int, int] | None: """Сохранённые трассы класса, который критерии требуют полным: у медленных записей — среди медленных по серверу, у неудачных загрузок — цельные цепочки.""" if name == "write_slow": return class_share( classes, variant, name, "server_slow_traces", "server_slow", ) if name == "upload_bad": return class_share(classes, variant, name, "chains") return class_share(classes, variant, name) def log_share( classes: list[Mapping[str, Any]], variant: str, name: str, *, server_slow: bool = False, ) -> tuple[int, int] | None: """Записи журнала класса, у которых `trace_id` ведёт на трассу в Tempo, из записей с `trace_id`; `server_slow` — у медленных записей только медленные по серверу (их правило выбора обязано сохранить).""" if server_slow and name == "write_slow": return class_share( classes, variant, name, "server_slow_linked", "server_slow", ) return class_share( classes, variant, name, "log_to_trace", "logs_with_trace", ) def quantile_error(rows: list[Mapping[str, Any]]) -> dict[str, float | None]: """Медиана по повторам относительной ошибки p95 и p99 из buckets против журнала, по маршрутам (ошибка считается в каждом прогоне, а не по медианам квантилей).""" out = {} for route in ("read", "write"): for q in ("p95", "p99"): errors = [] for row in rows: b = num(row.get(f"q_{route}_{q}_buckets")) log = num(row.get(f"q_{route}_{q}_log")) if b is not None and log: errors.append(abs(b - log) / log) out[f"{route}_{q}"] = statistics.median(errors) if errors else None return out def verdicts( rows: list[Mapping[str, Any]], classes: list[Mapping[str, Any]], collector_memory_mib: float, ) -> list[Verdict]: g = by_variant(rows) out = [] cpu = "replica_cpu_ms_per_req" def m(variant: str, key: str) -> float | None: return med(g.get(variant, []), key) def eff(variant: str, reference: str, key: str) -> bool | None: return effect(g, variant, reference, key) # H1 h = Verdict("H1") full_cpu, off_cpu = m("full", cpu), m("off", cpu) r = ratio(full_cpu, off_cpu) h.need( "CPU реплик на запрос ≥ +10 %", None if r is None else bool(eff("full", "off", cpu)) and r >= 1.10, f"{fmt(full_cpu, 3)} против {fmt(off_cpu, 3)} мс", ) p50 = (m("full", "light_p50"), m("off", "light_p50")) p99 = (m("full", "light_p99"), m("off", "light_p99")) d50 = None if None in p50 else p50[0] - p50[1] d99 = None if None in p99 else p99[0] - p99[1] h.need( "p50 лёгких не больше +1 мс", None if d50 is None else d50 <= 1, f"Δ {fmt(d50)} мс", ) rel50, rel99 = ratio(*p50), ratio(*p99) h.need( "p99 лёгких ≥ +2 мс и растёт сильнее p50", None if d99 is None or rel50 is None or rel99 is None else d99 >= 2 and rel99 > rel50, f"Δ {fmt(d99)} мс, рост p99 ×{fmt(rel99)} против p50 ×{fmt(rel50)}", ) out.append(h) # H2 h = Verdict("H2") head_cpu = m("head", cpu) h.need( "надбавка CPU head ≤ половины надбавки full", None if None in (head_cpu, off_cpu, full_cpu) else head_cpu - off_cpu <= 0.5 * (full_cpu - off_cpu), f"head {fmt(head_cpu, 3)}, off {fmt(off_cpu, 3)}, full " f"{fmt(full_cpu, 3)} мс", ) tr = ratio(m("head", "tempo_spans_per_s"), m("full", "tempo_spans_per_s")) h.need( "приём Tempo ≤ 15 % full", None if tr is None else tr <= 0.15, f"{fmt(None if tr is None else tr * 100, 1)} %", ) head_ratio = 0.1 for name in HEAD_CLASSES: share = class_share(classes, "head", name) if share is None or not share[1]: h.need(f"доля трасс {name} около 0,1", None) continue lo, hi = binom_interval(*share) h.need( f"доля трасс {name} около 0,1", lo <= head_ratio <= hi, f"{share[0]}/{share[1]}, интервал {lo:.3f}–{hi:.3f}", ) for name in HEAD_CLASSES: logs = class_share(classes, "head", name, "logs") h.need( f"записи журнала {name} — 100 %", None if logs is None else logs[0] == logs[1], "" if logs is None else f"{logs[0]}/{logs[1]}", ) # счёт в метрике — с начала процессов: у регистраций ещё пользователи # prepare.js, поэтому «не меньше клиента»; медленные записи server span # и клиент меряют по-разному — не сравниваются for name in ("broken", "register", "upload_bad"): metric = class_share(classes, "head", name, "metric") h.need( f"счёт {name} в метрике — 100 %", None if metric is None else metric[0] >= metric[1], "" if metric is None else f"{metric[0]}/{metric[1]}", ) out.append(h) # H3 h = Verdict("H3") tail_effect = eff("tail", "full", cpu) h.need( "CPU реплик на запрос в пределах шума от full", None if tail_effect is None else not tail_effect, f"tail {fmt(m('tail', cpu), 3)}, full {fmt(full_cpu, 3)} мс", ) peak = ( m("tail", "otel_collector_peak_mib"), m("full", "otel_collector_peak_mib"), ) h.need( "пик памяти Collector ≥ full + 15 МиБ", None if None in peak else peak[0] >= peak[1] + 15, f"{fmt(peak[0], 1)} против {fmt(peak[1], 1)} МиБ", ) ccpu = (m("tail", "otel_collector_cpu"), m("full", "otel_collector_cpu")) h.need( "CPU Collector не выше full больше чем на 20 %", None if None in ccpu else ccpu[0] <= ccpu[1] * 1.2, f"{fmt(ccpu[0], 3)} против {fmt(ccpu[1], 3)} ядра", ) tr = ratio(m("tail", "tempo_spans_per_s"), m("full", "tempo_spans_per_s")) h.need( "приём Tempo ≤ 30 % full", None if tr is None else tr <= 0.30, f"{fmt(None if tr is None else tr * 100, 1)} %", ) keep_all(h, classes, "tail", require_chain_bad=True) chains = class_share(classes, "tail", "upload", "chains") h.need( "цепочки обычных загрузок ≥ 90 %", None if chains is None or not chains[1] else chains[0] / chains[1] >= 0.9, "" if chains is None else f"{chains[0]}/{chains[1]}", ) refused = sum(values(g.get("tail", []), "col_span_lost")) + sum( values(g.get("tail", []), "tail_dropped_too_early"), ) h.need( "нет отказов memory_limiter и dropped_too_early", None if not g.get("tail") else refused == 0, f"{refused:g}", ) out.append(h) # H4 h = Verdict("H4") sr = ratio(m("metrics", "prom_series"), m("full", "prom_series")) h.need( "рядов Prometheus ≤ 40 % full", None if sr is None else sr <= 0.4, f"{fmt(None if sr is None else sr * 100, 1)} %", ) ar = ratio( m("metrics", "prom_samples_per_s"), m("full", "prom_samples_per_s"), ) h.need( "отсчётов в секунду ≤ 40 % full", None if ar is None else ar <= 0.4, f"{fmt(None if ar is None else ar * 100, 1)} %", ) for key, label in ( (cpu, "CPU реплик"), ("otel_collector_cpu", "CPU Collector"), ): e = eff("metrics", "full", key) h.need(f"{label} в пределах шума", None if e is None else not e) coarse = quantile_error(g.get("metrics", [])) fine = quantile_error(g.get("full", [])) pairs = [ (coarse[k], fine[k]) for k in coarse if coarse[k] is not None and fine[k] is not None ] h.need( "ошибка квантиля coarse ≥ 2 × default хотя бы на одном маршруте", None if not pairs else any(c >= 2 * max(f, 1e-9) for c, f in pairs), ", ".join( f"{k} {fmt(coarse[k], 3)} / {fmt(fine[k], 3)}" for k in coarse ), ) out.append(h) # H5 h = Verdict("H5") # full — все редкие классы; tail и balanced — классы, которые их правила # сохраняют (медленные записи — медленные по серверу) for variant, names in ( ("full", HEAD_CLASSES), ("tail", MUST_KEEP), ("balanced", MUST_KEEP), ): for name in names: share = log_share( classes, variant, name, server_slow=variant != "full", ) if share is None or not share[1]: h.need(f"{variant}: журнал → трасса {name} — 100 %", None) continue h.need( f"{variant}: журнал → трасса {name} — 100 %", share[0] == share[1], f"{share[0]}/{share[1]}", ) for name in HEAD_CLASSES: share = log_share(classes, "head", name) if share is None or not share[1]: h.need(f"head: журнал → трасса {name} около 0,1", None) continue lo, hi = binom_interval(*share) h.need( f"head: журнал → трасса {name} около 0,1", lo <= head_ratio <= hi, f"{share[0]}/{share[1]}, интервал {lo:.3f}–{hi:.3f}", ) out.append(h) # H6 h = Verdict("H6") telemetry_rows = [r for r in rows if r["variant"] != "off"] lost = sum(sum(values(telemetry_rows, key)) for key in LOSS_KEYS) h.need( "потерь SDK и Collector и dropped_too_early нет", None if not telemetry_rows else lost == 0, f"{lost:g}", ) gaps = values(telemetry_rows, "spans_gap") h.need( "span SDK → Collector → Tempo сходятся с точностью 1 % (выборка " "отдельно)", None if not gaps else max(gaps) <= 0.01, f"наибольшее расхождение {fmt(max(gaps) * 100 if gaps else None)} %", ) out.append(h) # B1–B5 и H7 bs = criteria(g, classes, collector_memory_mib) h = Verdict("H7") for b in bs: h.need( f"{b.name} — {b.result}", None if b.result == "нет данных" else b.result == "подтверждено", ) out.append(h) # H8 h = Verdict("H8") t, lk, pr = ( m("full", k) for k in ("tempo_cpu", "loki_cpu", "prometheus_cpu") ) h.need( "у full CPU Tempo больше Loki и Prometheus вместе", None if None in (t, lk, pr) else t > lk + pr, f"{fmt(t, 3)} против {fmt(lk, 3)} + {fmt(pr, 3)}", ) dt, dl, dp = (m("full", f"disk_{k}_bytes") for k in DISKS[::-1]) h.need( "рост Tempo на диске больше Loki и Prometheus вместе", None if None in (dt, dl, dp) else dt > dl + dp, f"{fmt(dt, 0)} против {fmt(dl, 0)} + {fmt(dp, 0)} байт", ) for variant in ("head", "tail", "balanced"): v = m(variant, "tempo_cpu") h.need( f"CPU Tempo у {variant} ≤ половины full", None if None in (v, t) else v <= 0.5 * t, f"{fmt(v, 3)} против {fmt(t, 3)}", ) out.append(h) return [*out, *bs] def keep_all( h: Verdict, classes: list[Mapping[str, Any]], variant: str, *, require_chain_bad: bool, ) -> None: labels = { "broken": "трассы 500 API — 100 %", "write_slow": "трассы медленных по серверу записей — 100 %", "upload_bad": "цепочки неудачных загрузок — 100 %", } for name in MUST_KEEP: if name == "upload_bad" and not require_chain_bad: continue share = kept(classes, variant, name) h.need( labels[name], None if share is None else share[0] == share[1], "" if share is None else f"{share[0]}/{share[1]}", ) # регистрация — редкий быстрый маршрут: сохраняется с долей остальных, # в вердикт не входит, показывается числом share = class_share(classes, variant, "register") if share is not None: h.notes.append( f"регистрации (редкий быстрый маршрут, справочно) {share[0]}/" f"{share[1]}", ) def criteria( g: Mapping[str, list[Mapping[str, Any]]], classes: list[Mapping[str, Any]], collector_memory_mib: float, ) -> list[Verdict]: """Критерии сбалансированной конфигурации B1–B5 (спек `load-benchmark`, «Цена телеметрии в отчёте»).""" def m(variant: str, key: str) -> float | None: return med(g.get(variant, []), key) out = [] b = Verdict("B1") keep_all(b, classes, "balanced", require_chain_bad=True) out.append(b) b = Verdict("B2") tb, tf, tt = ( m(v, "tempo_spans_per_s") for v in ("balanced", "full", "tail") ) b.need( "приём Tempo ≤ 25 % full", None if None in (tb, tf) or not tf else tb / tf <= 0.25, f"{fmt(ratio(tb, tf) and ratio(tb, tf) * 100, 1)} %", ) b.need( "приём Tempo меньше, чем у tail", None if None in (tb, tt) else tb < tt, f"{fmt(tb, 1)} против {fmt(tt, 1)} span/с", ) sr = ratio(m("balanced", "prom_series"), m("full", "prom_series")) b.need( "рядов Prometheus ≤ 40 % full", None if sr is None else sr <= 0.4, f"{fmt(None if sr is None else sr * 100, 1)} %", ) out.append(b) b = Verdict("B3") rows = g.get("balanced", []) lost = sum(sum(values(rows, key)) for key in LOSS_KEYS) b.need( "потерь и отброшенных до решения нет", None if not rows else lost == 0, f"{lost:g}", ) peaks = values(rows, "otel_collector_peak_mib") b.need( "пик памяти Collector ≤ 2/3 лимита", None if not peaks else max(peaks) <= collector_memory_mib * 2 / 3, f"{fmt(max(peaks) if peaks else None, 1)} из " f"{collector_memory_mib:g} МиБ", ) out.append(b) b = Verdict("B4") cb, co = ( m("balanced", "replica_cpu_ms_per_req"), m( "off", "replica_cpu_ms_per_req", ), ) b.need( "CPU реплик на запрос не больше +25 %", None if None in (cb, co) or not co else cb / co <= 1.25, f"{fmt(cb, 3)} против {fmt(co, 3)} мс", ) for key, limit in (("light_p95", 3), ("light_p99", 10)): a, z = m("balanced", key), m("off", key) b.need( f"{key} не больше +{limit} мс", None if None in (a, z) else a - z <= limit, f"Δ {fmt(None if None in (a, z) else a - z)} мс", ) out.append(b) b = Verdict("B5") for name in MUST_KEEP: share = log_share(classes, "balanced", name, server_slow=True) b.need( f"журнал → трасса {name}", None if share is None else share[0] == share[1], "" if share is None else f"{share[0]}/{share[1]}", ) out.append(b) return out # ------------------------------------------------------------- сводка ORDER = ("off", "full", "full-aa", "head", "tail", "metrics", "balanced") def spread(rows: list[Mapping[str, Any]], key: str, digits: int = 2) -> str: return reconcile.spread((r.get(key) for r in rows), digits) def render_summary( meta: Mapping[str, Any], rows: list[Mapping[str, Any]], classes: list[Mapping[str, Any]], comps: list[Mapping[str, Any]], ) -> str: g = by_variant(rows) names = [n for n in ORDER if n in g] profile = meta.get("profile", {}) info = meta.get("telemetry_cost", {}) collector_mib = ( reconcile_size(profile.get("collector_memory", "256m")) / 2**20 ) lines = [ f"# Цена телеметрии — метка {meta.get('label')}", "", f"Commit `{meta.get('git', {}).get('commit')}`, dirty " f"{meta.get('git', {}).get('dirty')}; профиль {profile.get('name')}; " f"окна {info.get('windows')} с; повторов {profile.get('repeats')}; " f"статус {meta.get('status')}.", "", "Медиана и разброс (min–max) повторов; шум — |full − full-aa|.", "", "## Задержки лёгких запросов (чистое окно), мс", "", "| Вариант | p50 | p95 | p99 | ошибок |", "|---|---|---|---|---|", ] for name in names: r = g[name] lines.append( f"| {name} | {spread(r, 'light_p50')} | {spread(r, 'light_p95')} " f"| {spread(r, 'light_p99')} | {spread(r, 'light_errors', 0)} |", ) lines += [ "", "## Остальные классы (чистое окно): p95, мс", "", "| Вариант | " + " | ".join(CLASSES) + " |", "|---|" + "---|" * len(CLASSES), ] for name in names: lines.append( f"| {name} | " + " | ".join(spread(g[name], f"{c}_p95", 1) for c in CLASSES) + " |", ) lines += [ "", "## CPU и память", "", "| Вариант | CPU реплик на запрос, мс | реплики, ядра | приложение " "| конвейер телеметрии | прочее | Collector пик, МиБ |", "|---|---|---|---|---|---|---|", ] for name in names: r = g[name] lines.append( f"| {name} | {spread(r, 'replica_cpu_ms_per_req', 3)} | " f"{spread(r, 'replica_cpu_cores', 3)} | " f"{spread(r, 'app_cpu_cores', 3)} | " f"{spread(r, 'telemetry_cpu_cores', 3)} | " f"{spread(r, 'rest_cpu_cores', 3)} | " f"{spread(r, 'otel_collector_peak_mib', 1)} |", ) noise_cpu = noise(g, "replica_cpu_ms_per_req") lines += [ "", f"Шум CPU реплик на запрос (A/A): {noise_cpu:.4f} мс.", "", "### По компонентам: CPU, ядра (медиана), пик памяти, МиБ", "", "| Вариант | " + " | ".join(f"{role} CPU / пик" for role in TELEMETRY_ROLES) + " |", "|---|" + "---|" * len(TELEMETRY_ROLES), ] for name in names: r = g[name] cells = [] for role in TELEMETRY_ROLES: p = role.replace("-", "_") cells.append( f"{fmt(med(r, f'{p}_cpu'), 3)} / " f"{fmt(med(r, f'{p}_peak_mib'), 1)}", ) lines.append(f"| {name} | " + " | ".join(cells) + " |") lines += [ "", "## Приём по сигналам (чистое окно, в секунду) и рост на диске", "", "| Вариант | span Collector | записей | точек | span Tempo | байт Tempo " "| строк Loki | байт Loki | отсчётов Prometheus | рядов | диск " "Prometheus / Loki / Tempo, МиБ |", "|---|---|---|---|---|---|---|---|---|---|---|", ] for name in names: r = g[name] disk = " / ".join( fmt( None if med(r, f"disk_{d}_bytes") is None else med(r, f"disk_{d}_bytes") / 2**20, 1, ) for d in DISKS ) lines.append( f"| {name} | {spread(r, 'col_accepted_spans_per_s', 0)} | " f"{spread(r, 'col_accepted_logs_per_s', 0)} | " f"{spread(r, 'col_accepted_points_per_s', 0)} | " f"{spread(r, 'tempo_spans_per_s', 0)} | " f"{spread(r, 'tempo_bytes_per_s', 0)} | " f"{spread(r, 'loki_lines_per_s', 0)} | " f"{spread(r, 'loki_bytes_per_s', 0)} | " f"{spread(r, 'prom_samples_per_s', 0)} | " f"{spread(r, 'prom_series', 0)} | {disk} |", ) run_s = sum(info.get("windows") or [0]) lines += [ "", "Пересчёт на сутки — приём в секунду × 86 400; срок хранения 7 суток " "— × 604 800 (рост на диске за прогон — журнал предзаписи, без " f"сжатых блоков: прогон {run_s} с).", "", "## Потери по звеньям и отброшенное выборкой (весь прогон)", "", "| Вариант | span SDK выбрано / отброшено в голове | SDK отправил | " "Collector принял / отправил | решения tail: сохр. / нет | Tempo " "принял | потери SDK span / записей | потери Collector span / " "записей | dropped_too_early | расхождение |", "|---|---|---|---|---|---|---|---|---|---|", ] for name in names: r = g[name] lines.append( f"| {name} | {spread(r, 'sdk_spans_sampled', 0)} / " f"{spread(r, 'sdk_spans_dropped', 0)} | " f"{spread(r, 'sdk_spans_exported', 0)} | " f"{spread(r, 'col_spans_accepted', 0)} / " f"{spread(r, 'col_spans_sent', 0)} | " f"{spread(r, 'tail_sampled', 0)} / " f"{spread(r, 'tail_not_sampled', 0)} | " f"{spread(r, 'tempo_spans_total', 0)} | " f"{spread(r, 'sdk_span_lost', 0)} / {spread(r, 'sdk_log_lost', 0)} " f"| {spread(r, 'col_span_lost', 0)} / " f"{spread(r, 'col_log_lost', 0)} | " f"{spread(r, 'tail_dropped_too_early', 0)} | " f"{spread(r, 'spans_gap', 4)} |", ) lines += [ "", "## Время до ready и итоги заданий", "", "| Вариант | p50, с | p95, с | принято / ready | bad принято / " "failed | медленных записей | недосланных |", "|---|---|---|---|---|---|---|", ] for name in names: r = g[name] lines.append( f"| {name} | {spread(r, 'ttr_p50_s', 3)} | " f"{spread(r, 'ttr_p95_s', 3)} | " f"{int(sum(values(r, 'accepted')))} / " f"{int(sum(values(r, 'ready')))} | " f"{int(sum(values(r, 'bad_accepted')))} / " f"{int(sum(values(r, 'bad_failed')))} | " f"{int(sum(values(r, 'slow_writes')))} | " f"{int(sum(values(r, 'dropped')))} |", ) lost_runs = [r["run"] for r in rows if "lost" in (r.get("flags") or "")] if lost_runs: lines += ["", f"Принятая работа не дошла до итога: {lost_runs}."] lines += [ "", "## Что сохраняет каждый вариант (сумма повторов)", "", "Трасса по `X-Request-Id`, цепочка загрузки (server span API, " "producer relay, consumer worker с итогом), запись журнала, переход " "журнал → трасса, счёт в метрике (конец прогона).", "", "| Вариант | Класс | по клиенту | трасс | цепочек | записей | с " "trace_id | журнал → трасса | метрика | медленных по серверу / их " "трасс |", "|---|---|---|---|---|---|---|---|---|---|", ] for name in names: for cls in RARE: rows_c = [ r for r in classes if r["variant"] == name and r["class"] == cls ] if not rows_c: continue def total( key: str, items: list[Mapping[str, Any]] = rows_c, ) -> str: found = values(items, key) return "—" if not found else f"{int(sum(found))}" lines.append( f"| {name} | {cls} | {total('client')} | {total('traces')} | " f"{total('chains')} | {total('logs')} | " f"{total('logs_with_trace')} | {total('log_to_trace')} | " f"{total('metric')} | {total('server_slow')} / " f"{total('server_slow_traces')} |", ) lines += [ "", "## Точность квантилей: из buckets против журнала (чистое окно), с", "", "| Вариант | read p95 | read p99 | write p95 | write p99 |", "|---|---|---|---|---|", ] for name in names: r = g[name] cells = [] for route in ("read", "write"): for q in ("p95", "p99"): cells.append( f"{fmt(med(r, f'q_{route}_{q}_buckets'), 4)} / " f"{fmt(med(r, f'q_{route}_{q}_log'), 4)}", ) lines.append(f"| {name} | " + " | ".join(cells) + " |") lines += ["", "## Гипотезы и критерии", ""] for verdict in verdicts(rows, classes, collector_mib): lines += [verdict.render(), ""] flagged = [f"{r['run']}: {r['flags']}" for r in rows if r.get("flags")] lines += ["## Пометки прогонов", ""] lines += [f"- {item}" for item in flagged] or ["нет"] warnings = meta.get("host_warnings") or [] if warnings: lines += ["", "## Предупреждения о хосте", ""] lines += [f"- {w}" for w in warnings] lines += [ "", "Графики: `cpu-by-group.svg`, `latency-light.svg`, `ingest.svg`, " "`traces-by-class.svg`.", "", ] del comps return "\n".join(lines) def reconcile_size(text: str) -> int: units = {"k": 2**10, "m": 2**20, "g": 2**30} text = str(text).strip().lower() if text and text[-1] in units: return int(float(text[:-1]) * units[text[-1]]) return int(float(text)) def grouped_bars( title: str, unit: str, groups: list[str], series: list[tuple[str, list[float | None]]], *, stacked: bool = False, ) -> str: """Столбики по вариантам: рядом (или стопкой) несколько рядов.""" width, height = 860, 420 left, right, top, bottom = 70, 20, 60, 70 svg = plots.Svg(width, height, title) totals = [ sum(v or 0 for _, vals in series for v in [vals[i]]) if stacked else max((vals[i] or 0) for _, vals in series) for i in range(len(groups)) ] high = max(totals + [0.0]) or 1.0 ticks = plots.nice_ticks(high) top_value = ticks[-1] plot_h = height - top - bottom plot_w = width - left - right def y(value: float) -> float: return top + plot_h * (1 - value / top_value) for tick in ticks: svg.line(left, y(tick), width - right, y(tick)) svg.text(left - 6, y(tick) + 4, f"{tick:g}", anchor="end", size=11) svg.text(18, top + plot_h / 2, unit, rotate=True, anchor="middle", size=11) slot = plot_w / max(1, len(groups)) bars = 1 if stacked else max(1, len(series)) bar_w = min(40.0, slot * 0.7 / bars) for i, group in enumerate(groups): x0 = left + slot * i + (slot - bar_w * bars) / 2 base = 0.0 for j, (label, vals) in enumerate(series): value = vals[i] if value is None: continue color = plots.SERIES[j % len(plots.SERIES)] if stacked: svg.bar( x0, y(base + value), bar_w, y(base) - y(base + value), color, f"{group} {label}: {value:.3g}", top=j == len(series) - 1, ) base += value else: x = x0 + j * bar_w svg.bar( x, y(value), bar_w - 2, y(0) - y(value), color, f"{group} {label}: {value:.3g}", top=True, ) svg.text( left + slot * i + slot / 2, height - bottom + 18, group, anchor="middle", size=11, ) svg.legend( [ (plots.SERIES[j % len(plots.SERIES)], label, "box") for j, (label, _) in enumerate(series) ], left, height - 24, plot_w, ) return svg.render() def render_plots( rows: list[Mapping[str, Any]], classes: list[Mapping[str, Any]], ) -> dict[str, str]: g = by_variant(rows) names = [n for n in ORDER if n in g] def series(keys: list[tuple[str, str]]) -> list[tuple[str, list[Any]]]: return [ (label, [med(g[n], key) for n in names]) for key, label in keys ] out = { "cpu-by-group.svg": grouped_bars( "CPU по группам в чистом окне", "ядра", names, series( [ ("app_cpu_cores", "приложение"), ("telemetry_cpu_cores", "конвейер телеметрии"), ("rest_cpu_cores", "прочее"), ], ), stacked=True, ), "latency-light.svg": grouped_bars( "Лёгкие запросы: p50, p95, p99", "мс", names, series( [ ("light_p50", "p50"), ("light_p95", "p95"), ("light_p99", "p99"), ], ), ), } full = g.get("full", []) def relative(key: str) -> list[float | None]: base = med(full, key) return [ None if med(g[n], key) is None or not base else med(g[n], key) / base * 100 for n in names ] out["ingest.svg"] = grouped_bars( "Приём по сигналам, % от full", "%", names, [ ("span Tempo", relative("tempo_spans_per_s")), ("строк Loki", relative("loki_lines_per_s")), ("отсчётов Prometheus", relative("prom_samples_per_s")), ], ) shares = [] for cls in RARE: vals = [] for n in names: share = class_share(classes, n, cls) vals.append( None if share is None or not share[1] else share[0] / share[1] * 100, ) shares.append((cls, vals)) out["traces-by-class.svg"] = grouped_bars( "Сохранённые трассы редких событий, %", "%", names, shares, ) return out def read_csv(path: Path) -> list[dict[str, Any]]: return overload.read_csv(path) # ------------------------------------------------------------ selftest def selftest() -> None: table = variants(0.1, 30) assert set(table) == set(ORDER) assert sdk_env(table["off"]) == {} assert sdk_env(table["head"])["TELEMETRY__TRACE_SAMPLE_RATIO"] == "0.1" assert sdk_env(table["metrics"])["TELEMETRY__METRIC_BUCKETS"] == "coarse" assert wait_after(table["full"]) == 35 assert wait_after(table["metrics"]) == 50 assert wait_after(table["tail"]) == 52 assert wait_after(table["balanced"]) == 35 assert wait_after(table["off"]) == 0 compose = json.loads(variant_compose(table["balanced"], ROOT)) assert compose["services"]["otel-collector"]["command"][1].endswith( "layer.yaml", ) assert compose["services"]["prometheus"]["volumes"][0]["target"] == ( PROMETHEUS_TARGET ) assert "services" in json.loads(variant_compose(table["off"], ROOT)) plan = plan_runs(ORDER, 3, 20261114) assert len(plan) == 21 for a, b in zip(plan, plan[1:], strict=False): assert a["variant"] != b["variant"], (a, b) assert {p["variant"] for p in plan[:7]} == set(ORDER) base = (ROOT / PROMETHEUS_CONFIG).read_text(encoding="utf-8") lean = (ROOT / PROMETHEUS_LEAN).read_text(encoding="utf-8") assert lean_problems(lean, base) == [], lean_problems(lean, base) assert lean_problems(lean, base.replace("15s", "20s", 1)) assert lean_problems(lean.replace(LEAN_END, "", 1), base) base_cfg = { "processors": {"memory_limiter": {}, "batch": {}}, "service": { "pipelines": { "traces": {"processors": ["memory_limiter", "batch"]}, "logs": {"processors": ["memory_limiter", "batch"]}, }, }, } good = json.loads(json.dumps(base_cfg)) good["processors"]["tail_sampling"] = {"policies": []} good["service"]["pipelines"]["traces"]["processors"] = [ "memory_limiter", "tail_sampling", "batch", ] assert collector_layer_problems(base_cfg, good) == [] bad = json.loads(json.dumps(good)) bad["service"]["pipelines"]["logs"]["processors"].append("extra") assert any("logs" in p for p in collector_layer_problems(base_cfg, bad)) assert collector_layer_problems(base_cfg, base_cfg) records = { "api:a": [ { "event": "Started API", "telemetry_enabled": True, "telemetry_trace_sample_ratio": 0.1, "telemetry_metric_buckets": "default", "telemetry_metric_export_interval_seconds": 15, }, ], "worker": [{"event": "Started worker", "telemetry_enabled": False}], } found = startup_problems(records, table["head"]) assert found and all(p.startswith("worker:") for p in found), found assert startup_problems({"relay": []}, table["off"]) assert ( layer_problems( {"tail_sampling": True, "lean_prometheus": True}, table["balanced"], ) == [] ) assert layer_problems({"tail_sampling": True}, table["full"]) t0_ms = 1_800_000_000_000 lines = [f"t-t0,{t0_ms}"] reqs_spec = [ ("read", 35_000, 5.0, 200), ("read", 36_000, 7.0, 200), ("write", 37_000, 9.0, 201), ("write", 125_000, 400.0, 201), ("broken", 50_000, 4.0, 500), ("register", 60_000, 500.0, 201), ("upload", 40_000, 80.0, 202), ("upload_bad", 41_000, 70.0, 202), ("me", 45_000, 3.0, 200), ("me", 46_000, 3.0, 401), ] for n, (cls, offset, dur, status) in enumerate(reqs_spec): aid = f"a{n}" if cls.startswith("upload") else "-" lines.append( f"t,{cls},{t0_ms + offset},{dur},{status},r{n:031x},inst{n % 2}," f"{aid}", ) lines.append("garbage") t0, reqs = parse_k6("\n".join(lines)) assert t0 == t0_ms / 1000 and len(reqs) == len(reqs_spec) bounds = window_bounds(t0, (30, 90, 30)) clean = inside(reqs, bounds["clean"]) assert len(clean) == 9, len(clean) light = latency(clean, LIGHT) assert light["n"] == 5 and light["errors"] == 1 and light["p99"] == 9.0 ids = rare_ids(reqs, 250) assert len(ids["write_slow"]) == 1 and len(ids["broken"]) == 1 lo, hi = binom_interval(3, 30) assert lo < 0.1 < hi, (lo, hi) lo, hi = binom_interval(30, 30) assert hi == 1.0 and lo > 0.88 lo, hi = binom_interval(0, 30) assert lo == 0.0 and 0.1 < hi < 0.13, hi assert parse_du("100\t/v/prometheus\n200 /v/tempo\nx") == { "prometheus": 100, "tempo": 200, } ns = int(t0 * 1e9) sampler = [] for second in range(0, 160, 1): t = ns + second * 10**9 sampler.append( f"cg {t} - aaaaaaaaaaaa {second * 400_000} {second * 10} 0 0 " f"{100 * 2**20} 0", ) sampler.append( f"cg {t} - cccccccccccc {second * 100_000} {second * 10} " f"{second} 0 {50 * 2**20} 0", ) sampler.append(f"vm {t} 10 20 0") sampler.append(f"mp {t} cccccccccccc {80 * 2**20} {60 * 2**20} 0") meta = overload.Meta(200, "ready", "-", 3, t0 + 41) metas = { "a6": meta, "a7": overload.Meta(200, "failed", "invalid_image", 0, t0 + 42), } lookup = { "classes": { "broken": { "traces": 1, "logs": 1, "logs_with_trace": 1, "log_to_trace": 1, }, }, "totals": { "sdk_span_exported_ok": 1000, "col_accepted_spans": 1000, "col_sent_spans": 1000, "tempo_spans": 995, "sdk_span_processed_lost": 0, }, "clean": { "tempo_spans": {"delta": 900, "seconds": 90}, "prom_series": {"last": 5000}, }, "rare_metrics": {"broken": 1}, "quantiles": {"read": {"p95_buckets": 0.01, "p95_log": 0.007}}, "layers": {"tail_sampling": False, "lean_prometheus": False}, } src = Sources( run={"run": "full-r1", "variant": "full", "repeat": 1, "order": 1}, variant=table["full"], windows=(30, 90, 30), t0=t0, reqs=reqs, dropped=0, sampler_text="\n".join(sampler), containers={ "aaaaaaaaaaaa": {"name": "mtusi-bench-api-1", "role": "api"}, "cccccccccccc": { "name": "mtusi-bench-otel-collector", "role": "otel-collector", }, }, metas=metas, lookup=lookup, du={"prometheus": 1, "loki": 2, "tempo": 3}, slow_ms=250, ) comps = components(src) assert len(comps) == 2 api = next(c for c in comps if c["role"] == "api") assert abs(api["cpu_cores"] - 0.4) < 1e-6, api col = next(c for c in comps if c["role"] == "otel-collector") assert col["peak_mib"] == 80.0 and col["throttle"] == 0.1, col row = run_row(src, comps) assert set(row) == set(RUN_FIELDS), set(RUN_FIELDS) ^ set(row) assert row["tempo_spans_per_s"] == 10.0 assert row["ready"] == 1 and row["bad_failed"] == 1 assert row["spans_gap"] == 0.005 assert "lost" not in row["flags"] assert row["q_read_p95_buckets"] == 0.01 assert row["replica_cpu_ms_per_req"] is not None crow = class_rows(src) assert {r["class"] for r in crow} == set(RARE) events = events_for_lookup( reqs, t0, (30, 90, 30), slow_ms=250, sample_me=10, seed=1, collected_at=t0 + 300, ) assert events["classes"]["write_slow"] and len(events["sample_me"]) == 1 rows = [] classes = [] rng = random.Random(7) for variant in ORDER: for repeat in (1, 2, 3): base_cpu = {"off": 1.0, "head": 1.05}.get(variant, 1.2) rows.append( { "run": f"{variant}-r{repeat}", "variant": variant, "repeat": repeat, "replica_cpu_ms_per_req": base_cpu + rng.random() * 0.01, "light_p50": 3.0, "light_p95": 6.0, "light_p99": 9.0 if variant == "off" else 12.0, "tempo_spans_per_s": { "head": 10, "tail": 20, "balanced": 15, }.get(variant, 100), "prom_series": 1500 if variant in ("metrics", "balanced") else 5000, "prom_samples_per_s": 30 if variant in ("metrics", "balanced") else 100, "otel_collector_peak_mib": 90 if variant in ("tail", "balanced") else 60, "otel_collector_cpu": 0.1, "tempo_cpu": 0.2 if variant in ("full", "full-aa", "metrics") else 0.05, "loki_cpu": 0.05, "prometheus_cpu": 0.05, "disk_tempo_bytes": 3e6, "disk_loki_bytes": 1e6, "disk_prometheus_bytes": 1e6, "spans_gap": 0.001, "q_read_p95_buckets": 0.02 if variant == "metrics" else 0.0075, "q_read_p95_log": 0.007, **dict.fromkeys(LOSS_KEYS, 0), "flags": "", }, ) for cls in RARE: client = 30 keep = 3 if variant == "head" else client classes.append( { "variant": variant, "class": cls, "client": client, "traces": keep, "chains": keep, "logs": client, "logs_with_trace": client, "log_to_trace": keep, "metric": client, "server_slow": client, "server_slow_traces": keep, "server_slow_linked": keep, }, ) result = {v.name: v.result for v in verdicts(rows, classes, 256)} assert result["B1"] == "подтверждено", result assert result["B3"] == "подтверждено", result assert result["H2"] == "подтверждено", [ v.render() for v in verdicts(rows, classes, 256) if v.name == "H2" ] assert result["H4"] == "подтверждено", [ v.render() for v in verdicts(rows, classes, 256) if v.name == "H4" ] assert result["H8"] == "подтверждено", result text = render_summary( { "label": "self", "profile": {"name": "telemetry", "repeats": 3}, "telemetry_cost": {"windows": [30, 90, 30]}, }, rows, classes, [], ) assert "## Гипотезы и критерии" in text and "**B5:" in text for name, svg in render_plots(rows, classes).items(): assert svg.startswith("