Base accepting the saturated full-r3 run on the user's 2026-10-31 host warning rule instead of the swap decision and ask whether to re-shoot; complete the report section with limits, metric freshness at 30 s and per-class and per-component tables; correct derived numbers and the quantile error wording; align H6 in design with the report; fix the quantile_error docstring; park S1 and S4 as later proposals. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2319 lines
81 KiB
Python
2319 lines
81 KiB
Python
"""Профиль цены телеметрии без docker: варианты, план, разбор, вердикты.
|
||
|
||
Design change `track1-telemetry-cost`, решения 2, 6, 7, 9–12; оркестратор —
|
||
`bench/run.py` (профили `telemetry`, `telemetry_quick`). Только stdlib, `src`
|
||
не импортирует. Здесь — таблица вариантов, план кругами, разбор строк k6
|
||
(`k6/telemetry.js`), сэмплера `cost`, обхода метаданных, выборки
|
||
(`telemetry_lookup.py`) и размера volume; окна; CPU на запрос, приём, рост на
|
||
диске, потери по звеньям, доли классов редких событий, точность квантилей;
|
||
строки `runs.csv`, `components.csv`, `classes.csv`; правило шума, точный
|
||
биномиальный интервал, вердикты H1–H8 и B1–B5, пометки; `summary.md` и четыре
|
||
SVG; проверки конфигураций до стенда (слой Collector, копия Prometheus) и
|
||
записей о старте процессов.
|
||
|
||
uv run python bench/telemetry_cost.py --selftest
|
||
"""
|
||
|
||
import json
|
||
import math
|
||
import random
|
||
import statistics
|
||
import sys
|
||
from collections.abc import Iterable, Mapping
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import overload
|
||
import plots
|
||
import reconcile
|
||
import scaling
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
COLLECTOR_CONFIG = "infra/otel/collector.yaml"
|
||
PROMETHEUS_CONFIG = "infra/otel/prometheus/prometheus.yml"
|
||
TAIL_TYPICAL = "bench/otel/tail-typical.yaml"
|
||
TAIL_BALANCED = "bench/otel/tail-balanced.yaml"
|
||
PROMETHEUS_LEAN = "bench/otel/prometheus-lean.yml"
|
||
LEAN_BEGIN = "# bench-lean: begin"
|
||
LEAN_END = "# bench-lean: end"
|
||
# пути конфигураций в контейнерах (docker-compose.yml)
|
||
COLLECTOR_TARGET = "/etc/otelcol-contrib/config.yaml"
|
||
LAYER_TARGET = "/etc/otelcol-contrib/layer.yaml"
|
||
PROMETHEUS_TARGET = "/etc/prometheus/prometheus.yml"
|
||
# сроки ожидания решения выбора трасс в Collector, с (decision_wait слоёв)
|
||
DECISION_WAIT_S = {TAIL_TYPICAL: 30, TAIL_BALANCED: 10}
|
||
SCRAPE_S = 15
|
||
EXPORT_MARGIN_S = 5
|
||
DECISION_MARGIN_S = 2
|
||
|
||
# классы запросов (решение 8); лёгкие — для цены
|
||
LIGHT = ("read", "write", "me", "card")
|
||
CLASSES = (
|
||
"read",
|
||
"write",
|
||
"me",
|
||
"card",
|
||
"download",
|
||
"upload",
|
||
"upload_bad",
|
||
"broken",
|
||
"register",
|
||
)
|
||
OK_STATUS = {
|
||
"read": 200,
|
||
"write": 201,
|
||
"me": 200,
|
||
"card": 200,
|
||
"download": 200,
|
||
"upload": 202,
|
||
"upload_bad": 202,
|
||
"broken": 500,
|
||
"register": 201,
|
||
}
|
||
# классы редких событий: что ищется в хранилищах
|
||
RARE = ("broken", "write_slow", "register", "upload_bad", "upload")
|
||
# редкие события, которые выбор в голове сохраняет с долей около 0,1
|
||
HEAD_CLASSES = ("broken", "write_slow", "register", "upload_bad")
|
||
# сохранение, которое критерии tail и balanced требуют полным: ошибки,
|
||
# медленные по серверу записи (≥ порога по `duration_ms` записи доступа —
|
||
# той же величине, что server span) и цепочки неудачных загрузок.
|
||
# Регистрация — редкий, но быстрый маршрут (~85 мс на стенде, а не 0,3–1 с,
|
||
# как ждал design): у tail и balanced она сохраняется с долей остальных, это
|
||
# и есть «что теряется» (design, «Уточнения apply»)
|
||
MUST_KEEP = ("broken", "write_slow", "upload_bad")
|
||
ROLE_GROUPS = {
|
||
"app": ("api", "worker", "relay"),
|
||
"telemetry": ("otel-collector", "prometheus", "loki", "tempo"),
|
||
}
|
||
TELEMETRY_ROLES = ROLE_GROUPS["telemetry"]
|
||
COMPONENT_FIELDS = [
|
||
"run",
|
||
"variant",
|
||
"repeat",
|
||
"role",
|
||
"name",
|
||
"cpu_cores",
|
||
"throttle",
|
||
"mem_max_mib",
|
||
"peak_mib",
|
||
"anon_max_mib",
|
||
]
|
||
CLASS_FIELDS = [
|
||
"run",
|
||
"variant",
|
||
"repeat",
|
||
"class",
|
||
"client",
|
||
"traces",
|
||
"chains",
|
||
"logs",
|
||
"logs_with_trace",
|
||
"logs_sampled_flag",
|
||
"log_to_trace",
|
||
"metric",
|
||
"server_slow",
|
||
"server_slow_traces",
|
||
"server_slow_linked",
|
||
]
|
||
LATENCY_KEYS = ("n", "errors", "p50", "p95", "p99")
|
||
INGEST = (
|
||
"col_accepted_spans",
|
||
"col_accepted_logs",
|
||
"col_accepted_points",
|
||
"tempo_spans",
|
||
"tempo_bytes",
|
||
"loki_lines",
|
||
"loki_bytes",
|
||
"loki_sm_bytes",
|
||
"prom_samples",
|
||
)
|
||
DISKS = ("prometheus", "loki", "tempo")
|
||
LOSS_KEYS = (
|
||
"sdk_span_lost",
|
||
"sdk_log_lost",
|
||
"col_span_lost",
|
||
"col_log_lost",
|
||
"col_point_refused",
|
||
"tail_dropped_too_early",
|
||
)
|
||
RUN_FIELDS = [
|
||
"run",
|
||
"variant",
|
||
"repeat",
|
||
"order",
|
||
"warm_s",
|
||
"clean_s",
|
||
"fault_s",
|
||
*(f"{name}_{key}" for name in ("light", *CLASSES) for key in LATENCY_KEYS),
|
||
"replica_cpu_ms_per_req",
|
||
"replica_cpu_cores",
|
||
"app_cpu_cores",
|
||
"telemetry_cpu_cores",
|
||
"rest_cpu_cores",
|
||
*(
|
||
f"{role.replace('-', '_')}_{key}"
|
||
for role in TELEMETRY_ROLES
|
||
for key in ("cpu", "throttle", "mem_max_mib", "peak_mib")
|
||
),
|
||
*(f"{name}_per_s" for name in INGEST),
|
||
"prom_series",
|
||
*(f"disk_{name}_bytes" for name in DISKS),
|
||
"sdk_spans_sampled",
|
||
"sdk_spans_dropped",
|
||
"sdk_spans_exported",
|
||
"col_spans_accepted",
|
||
"col_spans_sent",
|
||
"tempo_spans_total",
|
||
"tail_sampled",
|
||
"tail_not_sampled",
|
||
*LOSS_KEYS,
|
||
"spans_gap",
|
||
"ttr_p50_s",
|
||
"ttr_p95_s",
|
||
"accepted",
|
||
"ready",
|
||
"bad_accepted",
|
||
"bad_failed",
|
||
"slow_writes",
|
||
"dropped",
|
||
*(
|
||
f"q_{route}_{q}_{src}"
|
||
for route in ("read", "write")
|
||
for q in ("p95", "p99")
|
||
for src in ("buckets", "log")
|
||
),
|
||
"flags",
|
||
]
|
||
# пометки, при которых код команды в конце метки ненулевой (решение 11)
|
||
FATAL_FLAGS = ("lost",)
|
||
# калибровка (решение 8): CPU реплики в чистом окне, предел Collector,
|
||
# Tempo и worker, p95 времени до ready
|
||
CAL_REPLICA = (0.30, 0.60)
|
||
CAL_LIMITED = 0.70
|
||
CAL_TTR_P95_S = 5.0
|
||
Z_ALPHA = 0.05
|
||
|
||
|
||
# ------------------------------------------------------------- варианты
|
||
|
||
|
||
def variants(head_ratio: float, metrics_interval: float) -> dict[str, Any]:
|
||
"""Таблица вариантов (решение 2): настройки SDK трёх процессов, слой
|
||
Collector и конфигурация Prometheus. Loki и Tempo — без изменений."""
|
||
full = {
|
||
"telemetry": True,
|
||
"ratio": 1.0,
|
||
"buckets": "default",
|
||
"interval": 15,
|
||
"collector_layer": None,
|
||
"prometheus": None,
|
||
}
|
||
return {
|
||
"off": {"telemetry": False},
|
||
"full": full,
|
||
"full-aa": dict(full),
|
||
"head": {**full, "ratio": head_ratio},
|
||
"tail": {**full, "collector_layer": TAIL_TYPICAL},
|
||
"metrics": {
|
||
**full,
|
||
"interval": metrics_interval,
|
||
"buckets": "coarse",
|
||
"prometheus": PROMETHEUS_LEAN,
|
||
},
|
||
"balanced": {
|
||
**full,
|
||
"collector_layer": TAIL_BALANCED,
|
||
"prometheus": PROMETHEUS_LEAN,
|
||
},
|
||
}
|
||
|
||
|
||
def sdk_env(variant: Mapping[str, Any]) -> dict[str, str]:
|
||
"""Окружение `TELEMETRY__*` реплик, worker и relay варианта (без
|
||
включения и адреса — их даёт `telemetry.compose.yml`)."""
|
||
if not variant.get("telemetry"):
|
||
return {}
|
||
return {
|
||
"TELEMETRY__TRACE_SAMPLE_RATIO": f"{variant['ratio']:g}",
|
||
"TELEMETRY__METRIC_BUCKETS": variant["buckets"],
|
||
"TELEMETRY__METRIC_EXPORT_INTERVAL_SECONDS": f"{variant['interval']:g}",
|
||
}
|
||
|
||
|
||
def wait_after(variant: Mapping[str, Any]) -> float:
|
||
"""Ожидание отправок после нагрузки (решение 9): последняя отправка
|
||
метрик + scrape и решение выбора трасс доходят до хранилищ раньше
|
||
выборки."""
|
||
if not variant.get("telemetry"):
|
||
return 0.0
|
||
metrics = variant["interval"] + SCRAPE_S + EXPORT_MARGIN_S
|
||
layer = variant.get("collector_layer")
|
||
# после решения span ещё уходят в Tempo, а счётчик Tempo доходит до
|
||
# Prometheus через scrape (пробный tail без scrape — расхождение 8 %)
|
||
decision = (
|
||
DECISION_WAIT_S[layer] + DECISION_MARGIN_S + SCRAPE_S + EXPORT_MARGIN_S
|
||
if layer
|
||
else 0
|
||
)
|
||
return float(max(metrics, decision))
|
||
|
||
|
||
def variant_compose(variant: Mapping[str, Any], root: Path) -> str:
|
||
"""`.work/variant.compose.yml` прогона (JSON — подмножество YAML):
|
||
окружение SDK трёх процессов, у Collector — второй `--config` и bind слоя,
|
||
у Prometheus — bind своей конфигурации на тот же путь (compose сливает
|
||
volumes по пути в контейнере)."""
|
||
services: dict[str, Any] = {}
|
||
env = sdk_env(variant)
|
||
for service in ("api", "worker", "relay"):
|
||
if env:
|
||
services[service] = {"environment": dict(env)}
|
||
layer = variant.get("collector_layer")
|
||
if layer:
|
||
services["otel-collector"] = {
|
||
"command": [
|
||
f"--config={COLLECTOR_TARGET}",
|
||
f"--config={LAYER_TARGET}",
|
||
],
|
||
"volumes": [bind(root / layer, LAYER_TARGET)],
|
||
}
|
||
lean = variant.get("prometheus")
|
||
if lean:
|
||
services["prometheus"] = {
|
||
"volumes": [bind(root / lean, PROMETHEUS_TARGET)],
|
||
}
|
||
return json.dumps({"services": services}, indent=1) + "\n"
|
||
|
||
|
||
def bind(source: Path, target: str) -> dict[str, Any]:
|
||
return {
|
||
"type": "bind",
|
||
"source": str(source),
|
||
"target": target,
|
||
"read_only": True,
|
||
"bind": {"create_host_path": False},
|
||
}
|
||
|
||
|
||
def plan_runs(
|
||
names: Iterable[str],
|
||
repeats: int,
|
||
seed: int,
|
||
) -> list[dict[str, Any]]:
|
||
"""Круги с перестановкой от зерна; круг не начинается с варианта, которым
|
||
кончился прошлый: повтор одного варианта подряд не идёт."""
|
||
items = list(names)
|
||
plan: list[dict[str, Any]] = []
|
||
previous = None
|
||
for round_no in range(1, repeats + 1):
|
||
order = list(items)
|
||
random.Random(seed + round_no).shuffle(order)
|
||
if len(order) > 1 and order[0] == previous:
|
||
order[0], order[1] = order[1], order[0]
|
||
for name in order:
|
||
plan.append(
|
||
{
|
||
"run": f"{name}-r{round_no}",
|
||
"variant": name,
|
||
"repeat": round_no,
|
||
"order": len(plan) + 1,
|
||
},
|
||
)
|
||
previous = order[-1] if order else None
|
||
return plan
|
||
|
||
|
||
# ------------------------------------------------- проверки конфигураций
|
||
|
||
|
||
def lean_problems(lean: str, base: str) -> list[str]:
|
||
"""Копия Prometheus без размеченных блоков совпадает с рабочей построчно
|
||
(решение 7)."""
|
||
kept: list[str] = []
|
||
inside = False
|
||
for number, line in enumerate(lean.splitlines(), start=1):
|
||
mark = line.strip()
|
||
if mark == LEAN_BEGIN:
|
||
if inside:
|
||
return [f"line {number}: nested {LEAN_BEGIN}"]
|
||
inside = True
|
||
continue
|
||
if mark == LEAN_END:
|
||
if not inside:
|
||
return [f"line {number}: {LEAN_END} without begin"]
|
||
inside = False
|
||
continue
|
||
if not inside:
|
||
kept.append(line)
|
||
if inside:
|
||
return [f"{LEAN_BEGIN} is not closed"]
|
||
problems = []
|
||
base_lines = base.splitlines()
|
||
for number, (a, b) in enumerate(zip(kept, base_lines, strict=False), 1):
|
||
if a != b:
|
||
problems.append(
|
||
f"outside bench-lean blocks line {number} differs from "
|
||
f"{PROMETHEUS_CONFIG}: {a!r} != {b!r}",
|
||
)
|
||
break
|
||
if not problems and len(kept) != len(base_lines):
|
||
problems.append(
|
||
f"outside bench-lean blocks {len(kept)} lines, "
|
||
f"{PROMETHEUS_CONFIG} {len(base_lines)}",
|
||
)
|
||
return problems
|
||
|
||
|
||
def first_difference(a: Any, b: Any, path: str = "") -> str | None:
|
||
if isinstance(a, dict) and isinstance(b, dict):
|
||
for key in sorted(set(a) | set(b)):
|
||
if key not in a or key not in b:
|
||
return f"{path}.{key}".lstrip(".")
|
||
found = first_difference(a[key], b[key], f"{path}.{key}")
|
||
if found is not None:
|
||
return found
|
||
return None
|
||
return None if a == b else (path.lstrip(".") or "<root>")
|
||
|
||
|
||
def collector_layer_problems(
|
||
base: Mapping[str, Any],
|
||
layered: Mapping[str, Any],
|
||
) -> list[str]:
|
||
"""Итоговая конфигурация со слоем отличается от рабочей только
|
||
процессором `tail_sampling` и списком процессоров pipeline трасс — базовым
|
||
с `tail_sampling` перед `batch` (решение 6)."""
|
||
layered = json.loads(json.dumps(layered))
|
||
problems = []
|
||
processors = layered.get("processors") or {}
|
||
if "tail_sampling" not in processors:
|
||
problems.append("layer has no processors.tail_sampling")
|
||
processors.pop("tail_sampling", None)
|
||
traces = layered.get("service", {}).get("pipelines", {}).get("traces", {})
|
||
base_list = list(base["service"]["pipelines"]["traces"]["processors"])
|
||
if "batch" in base_list:
|
||
expected = list(base_list)
|
||
expected.insert(expected.index("batch"), "tail_sampling")
|
||
else:
|
||
expected = [*base_list, "tail_sampling"]
|
||
if traces.get("processors") != expected:
|
||
problems.append(
|
||
f"traces processors {traces.get('processors')}, expected "
|
||
f"{expected}",
|
||
)
|
||
traces["processors"] = base_list
|
||
diff = first_difference(dict(base), layered)
|
||
if diff is not None:
|
||
problems.append(f"layer changes {diff}")
|
||
return problems
|
||
|
||
|
||
STARTED = {
|
||
"api": "Started API",
|
||
"worker": "Started worker",
|
||
"relay": "Started relay",
|
||
}
|
||
|
||
|
||
def startup_problems(
|
||
records_by_process: Mapping[str, list[dict[str, Any]]],
|
||
variant: Mapping[str, Any],
|
||
) -> list[str]:
|
||
"""Записи о старте реплик, worker и relay против варианта (решение 11):
|
||
включена ли телеметрия, доля, набор границ, интервал."""
|
||
problems = []
|
||
for process, records in records_by_process.items():
|
||
kind = process.split(":", 1)[0]
|
||
event = STARTED[kind]
|
||
started = [r for r in records if r.get("event") == event]
|
||
if not started:
|
||
problems.append(f"{process}: no «{event}» record")
|
||
continue
|
||
record = started[-1]
|
||
expected: dict[str, Any] = {
|
||
"telemetry_enabled": bool(variant.get("telemetry")),
|
||
}
|
||
if variant.get("telemetry"):
|
||
expected.update(
|
||
{
|
||
"telemetry_trace_sample_ratio": float(variant["ratio"]),
|
||
"telemetry_metric_buckets": variant["buckets"],
|
||
"telemetry_metric_export_interval_seconds": float(
|
||
variant["interval"],
|
||
),
|
||
},
|
||
)
|
||
for key, value in expected.items():
|
||
seen = record.get(key)
|
||
if isinstance(value, float):
|
||
ok = isinstance(seen, int | float) and math.isclose(
|
||
float(seen),
|
||
value,
|
||
)
|
||
else:
|
||
ok = seen == value
|
||
if not ok:
|
||
problems.append(
|
||
f"{process}: {key} {seen!r}, expected {value!r}",
|
||
)
|
||
return problems
|
||
|
||
|
||
def layer_problems(
|
||
layers: Mapping[str, bool],
|
||
variant: Mapping[str, Any],
|
||
) -> list[str]:
|
||
"""Слои Collector и Prometheus по выборке (решение 9)."""
|
||
problems = []
|
||
want_tail = bool(variant.get("collector_layer"))
|
||
if bool(layers.get("tail_sampling")) != want_tail:
|
||
problems.append(
|
||
f"tail sampling series {layers.get('tail_sampling')}, "
|
||
f"expected {want_tail}",
|
||
)
|
||
want_lean = bool(variant.get("prometheus"))
|
||
if bool(layers.get("lean_prometheus")) != want_lean:
|
||
problems.append(
|
||
f"lean Prometheus config {layers.get('lean_prometheus')}, "
|
||
f"expected {want_lean}",
|
||
)
|
||
return problems
|
||
|
||
|
||
# ------------------------------------------------------------- разбор
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Req:
|
||
cls: str
|
||
sent_ms: float
|
||
duration_ms: float
|
||
status: int
|
||
request_id: str
|
||
instance: str
|
||
attachment_id: str
|
||
|
||
@property
|
||
def answered(self) -> float:
|
||
return (self.sent_ms + self.duration_ms) / 1000
|
||
|
||
|
||
def parse_k6(text: str) -> tuple[float | None, list[Req]]:
|
||
"""Строки `t-t0,<мс>` и `t,<класс>,<отправка>,<длительность>,<статус>,
|
||
<X-Request-Id>,<X-Instance-Id>,<id>`."""
|
||
t0 = None
|
||
out = []
|
||
for raw in text.splitlines():
|
||
parts = raw.strip().split(",")
|
||
if parts[0] == "t-t0" and len(parts) == 2:
|
||
t0 = float(parts[1]) / 1000
|
||
elif parts[0] == "t" and len(parts) == 8:
|
||
out.append(
|
||
Req(
|
||
parts[1],
|
||
float(parts[2]),
|
||
float(parts[3]),
|
||
int(parts[4]),
|
||
parts[5],
|
||
parts[6],
|
||
parts[7],
|
||
),
|
||
)
|
||
return t0, out
|
||
|
||
|
||
def window_bounds(
|
||
t0: float,
|
||
windows: tuple[int, int, int],
|
||
) -> dict[str, tuple[float, float]]:
|
||
warm, clean, fault = windows
|
||
return {
|
||
"warm": (t0, t0 + warm),
|
||
"clean": (t0 + warm, t0 + warm + clean),
|
||
"fault": (t0 + warm + clean, t0 + warm + clean + fault),
|
||
"run": (t0, t0 + warm + clean + fault),
|
||
}
|
||
|
||
|
||
def inside(reqs: Iterable[Req], w: tuple[float, float]) -> list[Req]:
|
||
return [r for r in reqs if w[0] * 1000 <= r.sent_ms < w[1] * 1000]
|
||
|
||
|
||
def latency(reqs: list[Req], classes: Iterable[str]) -> dict[str, Any]:
|
||
names = set(classes)
|
||
chosen = [r for r in reqs if r.cls in names]
|
||
ok = [r.duration_ms for r in chosen if r.status == OK_STATUS[r.cls]]
|
||
return {
|
||
"n": len(chosen),
|
||
"errors": len(chosen) - len(ok),
|
||
"p50": rounded(reconcile.percentile(ok, 50), 2),
|
||
"p95": rounded(reconcile.percentile(ok, 95), 2),
|
||
"p99": rounded(reconcile.percentile(ok, 99), 2),
|
||
}
|
||
|
||
|
||
def rounded(value: float | None, digits: int = 3) -> float | None:
|
||
return None if value is None else round(value, digits)
|
||
|
||
|
||
def rare_ids(reqs: list[Req], slow_ms: float) -> dict[str, list[str]]:
|
||
"""Id запросов классов редких событий (по клиенту): 500 вложения без
|
||
объекта, медленные записи, регистрации, принятые загрузки `bad` и
|
||
обычные."""
|
||
|
||
def ids(cls: str, status: int) -> list[str]:
|
||
return [
|
||
r.request_id
|
||
for r in reqs
|
||
if r.cls == cls and r.status == status and r.request_id != "-"
|
||
]
|
||
|
||
return {
|
||
"broken": ids("broken", 500),
|
||
"write_slow": [
|
||
r.request_id
|
||
for r in reqs
|
||
if r.cls == "write"
|
||
and r.status == 201
|
||
and r.duration_ms >= slow_ms
|
||
and r.request_id != "-"
|
||
],
|
||
"register": ids("register", 201),
|
||
"upload_bad": ids("upload_bad", 202),
|
||
"upload": ids("upload", 202),
|
||
}
|
||
|
||
|
||
def parse_du(text: str) -> dict[str, int]:
|
||
"""`du -sb /v/<имя>` по строкам: имя volume → байт."""
|
||
out = {}
|
||
for line in text.splitlines():
|
||
parts = line.split()
|
||
if len(parts) == 2 and parts[0].isdigit():
|
||
out[parts[1].rstrip("/").rsplit("/", 1)[-1]] = int(parts[0])
|
||
return out
|
||
|
||
|
||
@dataclass
|
||
class Sources:
|
||
"""Всё снятое за прогон."""
|
||
|
||
run: Mapping[str, Any]
|
||
variant: Mapping[str, Any]
|
||
windows: tuple[int, int, int]
|
||
t0: float
|
||
reqs: list[Req]
|
||
dropped: int
|
||
sampler_text: str
|
||
containers: Mapping[str, Mapping[str, str]]
|
||
metas: Mapping[str, Any]
|
||
lookup: Mapping[str, Any] | None
|
||
du: Mapping[str, int]
|
||
slow_ms: float
|
||
not_drained: bool = False
|
||
storage_down: bool = False
|
||
oom: bool = False
|
||
swapouts: int | None = None
|
||
|
||
|
||
def components(src: Sources) -> list[dict[str, Any]]:
|
||
"""CPU, троттлинг и память каждого контейнера стенда в чистом окне."""
|
||
w0, w1 = window_bounds(src.t0, src.windows)["clean"]
|
||
samples = scaling.parse_samples(src.sampler_text)
|
||
mp = overload.parse_mp(src.sampler_text)
|
||
rows = []
|
||
for (_, ident), points in sorted(samples.cg.items()):
|
||
info = src.containers.get(ident)
|
||
if not info:
|
||
continue
|
||
lives = overload.split_lives(points)
|
||
cores = overload.role_cores([lives], w0, w1)
|
||
in_window = [p for p in points if w0 * 1e9 <= p[0] <= w1 * 1e9]
|
||
peaks = mp.get(ident, [])
|
||
rows.append(
|
||
{
|
||
"run": src.run["run"],
|
||
"variant": src.run["variant"],
|
||
"repeat": src.run["repeat"],
|
||
"role": info["role"],
|
||
"name": info["name"],
|
||
"cpu_cores": cores,
|
||
"throttle": overload.throttle_share(lives, w0, w1),
|
||
"mem_max_mib": (
|
||
round(max(p[5] for p in in_window) / 2**20, 1)
|
||
if in_window
|
||
else None
|
||
),
|
||
"peak_mib": (
|
||
round(max(p[1] for p in peaks) / 2**20, 1)
|
||
if peaks
|
||
else None
|
||
),
|
||
"anon_max_mib": (
|
||
round(max(p[2] for p in peaks) / 2**20, 1)
|
||
if peaks
|
||
else None
|
||
),
|
||
},
|
||
)
|
||
return rows
|
||
|
||
|
||
def group_cores(rows: list[dict[str, Any]]) -> dict[str, float | None]:
|
||
out: dict[str, float] = {"app": 0.0, "telemetry": 0.0, "rest": 0.0}
|
||
seen = False
|
||
for row in rows:
|
||
cores = row["cpu_cores"]
|
||
if cores is None or row["role"] in ("k6", "sampler"):
|
||
continue
|
||
seen = True
|
||
group = next(
|
||
(g for g, roles in ROLE_GROUPS.items() if row["role"] in roles),
|
||
"rest",
|
||
)
|
||
out[group] += cores
|
||
if not seen:
|
||
return dict.fromkeys(out)
|
||
return {key: round(value, 4) for key, value in out.items()}
|
||
|
||
|
||
def num(value: Any) -> float | None:
|
||
return reconcile.num(value)
|
||
|
||
|
||
def zero(value: Any) -> float:
|
||
return num(value) or 0.0
|
||
|
||
|
||
def losses(totals: Mapping[str, Any]) -> dict[str, float]:
|
||
"""Потери по звеньям (решение 9): SDK — отброшенное процессором и
|
||
неудачные отправки; Collector — отказ приёма, ошибки и отказы отправки.
|
||
Отброшенное выборкой (в голове и решения Collector) сюда не входит."""
|
||
return {
|
||
"sdk_span_lost": zero(totals.get("sdk_span_processed_lost"))
|
||
+ zero(totals.get("sdk_span_export_failed")),
|
||
"sdk_log_lost": zero(totals.get("sdk_log_processed_lost"))
|
||
+ zero(totals.get("sdk_log_export_failed")),
|
||
"col_span_lost": sum(
|
||
zero(totals.get(key))
|
||
for key in (
|
||
"col_refused_spans",
|
||
"col_failed_spans",
|
||
"col_send_failed_spans",
|
||
"col_enqueue_failed_spans",
|
||
)
|
||
),
|
||
"col_log_lost": sum(
|
||
zero(totals.get(key))
|
||
for key in (
|
||
"col_refused_logs",
|
||
"col_failed_logs",
|
||
"col_send_failed_logs",
|
||
"col_enqueue_failed_logs",
|
||
)
|
||
),
|
||
"col_point_refused": zero(totals.get("col_refused_points")),
|
||
"tail_dropped_too_early": zero(totals.get("tail_dropped_too_early")),
|
||
}
|
||
|
||
|
||
def spans_gap(totals: Mapping[str, Any]) -> float | None:
|
||
"""Доля, на которую span не сходятся по звеньям: SDK отправил → Collector
|
||
принял, Collector отправил → Tempo принял (наибольшая из двух)."""
|
||
pairs = (
|
||
("sdk_span_exported_ok", "col_accepted_spans"),
|
||
("col_sent_spans", "tempo_spans"),
|
||
)
|
||
gaps = []
|
||
for a, b in pairs:
|
||
va, vb = num(totals.get(a)), num(totals.get(b))
|
||
if va is None or vb is None or not va:
|
||
continue
|
||
gaps.append(abs(va - vb) / va)
|
||
return round(max(gaps), 4) if gaps else None
|
||
|
||
|
||
def time_to_ready(
|
||
reqs: list[Req],
|
||
metas: Mapping[str, Any],
|
||
) -> tuple[float | None, float | None]:
|
||
values = []
|
||
for r in reqs:
|
||
meta = metas.get(r.attachment_id)
|
||
if (
|
||
r.cls == "upload"
|
||
and r.status == 202
|
||
and meta is not None
|
||
and meta.status == "ready"
|
||
and meta.updated_at is not None
|
||
):
|
||
values.append(max(0.0, meta.updated_at - r.answered))
|
||
return (
|
||
rounded(reconcile.percentile(values, 50), 3),
|
||
rounded(reconcile.percentile(values, 95), 3),
|
||
)
|
||
|
||
|
||
def run_row(
|
||
src: Sources,
|
||
comps: list[dict[str, Any]],
|
||
) -> dict[str, Any]:
|
||
bounds = window_bounds(src.t0, src.windows)
|
||
clean = inside(src.reqs, bounds["clean"])
|
||
row: dict[str, Any] = {
|
||
key: src.run[key] for key in ("run", "variant", "repeat", "order")
|
||
}
|
||
row.update(
|
||
dict(zip(("warm_s", "clean_s", "fault_s"), src.windows, strict=True)),
|
||
)
|
||
for name, classes in (("light", LIGHT), *((c, (c,)) for c in CLASSES)):
|
||
for key, value in latency(clean, classes).items():
|
||
row[f"{name}_{key}"] = value
|
||
replicas = [c for c in comps if c["role"] == "api"]
|
||
replica_cores = (
|
||
sum(c["cpu_cores"] for c in replicas if c["cpu_cores"] is not None)
|
||
if replicas
|
||
else None
|
||
)
|
||
answered = sum(1 for r in clean if r.instance != "-")
|
||
clean_s = src.windows[1]
|
||
row["replica_cpu_cores"] = rounded(replica_cores, 4)
|
||
row["replica_cpu_ms_per_req"] = (
|
||
round(replica_cores * clean_s * 1000 / answered, 4)
|
||
if replica_cores is not None and answered
|
||
else None
|
||
)
|
||
groups = group_cores(comps)
|
||
for group in ("app", "telemetry", "rest"):
|
||
row[f"{group}_cpu_cores"] = groups[group]
|
||
for role in TELEMETRY_ROLES:
|
||
found = [c for c in comps if c["role"] == role]
|
||
prefix = role.replace("-", "_")
|
||
item = found[0] if found else {}
|
||
row[f"{prefix}_cpu"] = item.get("cpu_cores")
|
||
row[f"{prefix}_throttle"] = item.get("throttle")
|
||
row[f"{prefix}_mem_max_mib"] = item.get("mem_max_mib")
|
||
row[f"{prefix}_peak_mib"] = item.get("peak_mib")
|
||
lookup = src.lookup or {}
|
||
clean_counts = lookup.get("clean", {})
|
||
for name in INGEST:
|
||
item = clean_counts.get(name) or {}
|
||
delta, seconds = item.get("delta"), item.get("seconds")
|
||
row[f"{name}_per_s"] = (
|
||
round(delta / seconds, 3)
|
||
if delta is not None and seconds
|
||
else None
|
||
)
|
||
row["prom_series"] = (clean_counts.get("prom_series") or {}).get("last")
|
||
for name in DISKS:
|
||
row[f"disk_{name}_bytes"] = src.du.get(name)
|
||
totals = lookup.get("totals", {})
|
||
row["sdk_spans_sampled"] = totals.get("sdk_span_started_sampled")
|
||
row["sdk_spans_dropped"] = totals.get("sdk_span_started_dropped")
|
||
row["sdk_spans_exported"] = totals.get("sdk_span_exported_ok")
|
||
row["col_spans_accepted"] = totals.get("col_accepted_spans")
|
||
row["col_spans_sent"] = totals.get("col_sent_spans")
|
||
row["tempo_spans_total"] = totals.get("tempo_spans")
|
||
row["tail_sampled"] = totals.get("tail_sampled")
|
||
row["tail_not_sampled"] = totals.get("tail_not_sampled")
|
||
lost = losses(totals) if src.lookup else {}
|
||
for key in LOSS_KEYS:
|
||
row[key] = lost.get(key)
|
||
row["spans_gap"] = spans_gap(totals) if src.lookup else None
|
||
row["ttr_p50_s"], row["ttr_p95_s"] = time_to_ready(src.reqs, src.metas)
|
||
accepted = [r for r in src.reqs if r.cls == "upload" and r.status == 202]
|
||
bad = [r for r in src.reqs if r.cls == "upload_bad" and r.status == 202]
|
||
|
||
def final(items: list[Req], status: str, code: str | None) -> int:
|
||
count = 0
|
||
for r in items:
|
||
meta = src.metas.get(r.attachment_id)
|
||
if meta is None or meta.status != status:
|
||
continue
|
||
if code is not None and meta.error_code != code:
|
||
continue
|
||
count += 1
|
||
return count
|
||
|
||
row["accepted"] = len(accepted)
|
||
row["ready"] = final(accepted, "ready", None)
|
||
row["bad_accepted"] = len(bad)
|
||
row["bad_failed"] = final(bad, "failed", "invalid_image")
|
||
row["slow_writes"] = len(rare_ids(src.reqs, src.slow_ms)["write_slow"])
|
||
row["dropped"] = src.dropped
|
||
for route in ("read", "write"):
|
||
values = (lookup.get("quantiles") or {}).get(route) or {}
|
||
for q in ("p95", "p99"):
|
||
for source in ("buckets", "log"):
|
||
row[f"q_{route}_{q}_{source}"] = rounded(
|
||
num(values.get(f"{q}_{source}")),
|
||
5,
|
||
)
|
||
row["flags"] = " ".join(run_flags(src, row))
|
||
return row
|
||
|
||
|
||
def run_flags(src: Sources, row: Mapping[str, Any]) -> list[str]:
|
||
flags = []
|
||
if src.dropped:
|
||
flags.append("dropped")
|
||
if src.swapouts:
|
||
flags.append("host_swap")
|
||
if src.storage_down:
|
||
flags.append("storage_down")
|
||
if src.oom:
|
||
flags.append("oom")
|
||
if src.not_drained:
|
||
flags.append("not_drained")
|
||
if (
|
||
row["ready"] != row["accepted"]
|
||
or row["bad_failed"] != row["bad_accepted"]
|
||
):
|
||
flags.append("lost")
|
||
if src.lookup and any(row.get(key) for key in LOSS_KEYS):
|
||
flags.append("telemetry_loss")
|
||
if src.run["variant"] == "full" and calibration_problems(row):
|
||
flags.append("saturated")
|
||
return flags
|
||
|
||
|
||
def calibration_problems(row: Mapping[str, Any]) -> list[str]:
|
||
"""Границы калибровки решения 8 у варианта `full`."""
|
||
problems = []
|
||
cores = num(row.get("replica_cpu_cores"))
|
||
if cores is not None:
|
||
share = cores / 2
|
||
if not CAL_REPLICA[0] <= share <= CAL_REPLICA[1]:
|
||
problems.append(f"replica CPU {share:.0%} of 1 CPU (30–60 %)")
|
||
for key, limit_cpus in (
|
||
("otel_collector_cpu", 0.5),
|
||
("tempo_cpu", 0.5),
|
||
):
|
||
value = num(row.get(key))
|
||
if value is not None and value / limit_cpus >= CAL_LIMITED:
|
||
problems.append(f"{key} {value / limit_cpus:.0%} of its limit")
|
||
if row.get("dropped"):
|
||
problems.append(f"dropped iterations {row['dropped']}")
|
||
ttr = num(row.get("ttr_p95_s"))
|
||
if ttr is not None and ttr >= CAL_TTR_P95_S:
|
||
problems.append(f"time to ready p95 {ttr} s")
|
||
return problems
|
||
|
||
|
||
def class_rows(src: Sources) -> list[dict[str, Any]]:
|
||
ids = rare_ids(src.reqs, src.slow_ms)
|
||
lookup = src.lookup or {}
|
||
found = lookup.get("classes", {})
|
||
metrics = lookup.get("rare_metrics", {})
|
||
rows = []
|
||
for name in RARE:
|
||
item = found.get(name, {})
|
||
rows.append(
|
||
{
|
||
"run": src.run["run"],
|
||
"variant": src.run["variant"],
|
||
"repeat": src.run["repeat"],
|
||
"class": name,
|
||
"client": len(ids[name]),
|
||
"traces": item.get("traces"),
|
||
"chains": item.get("chains"),
|
||
"logs": item.get("logs"),
|
||
"logs_with_trace": item.get("logs_with_trace"),
|
||
"logs_sampled_flag": item.get("logs_sampled_flag"),
|
||
"log_to_trace": item.get("log_to_trace"),
|
||
"metric": metrics.get(name),
|
||
"server_slow": item.get("server_slow"),
|
||
"server_slow_traces": item.get("server_slow_traces"),
|
||
"server_slow_linked": item.get("server_slow_linked"),
|
||
},
|
||
)
|
||
return rows
|
||
|
||
|
||
def events_for_lookup(
|
||
src_reqs: list[Req],
|
||
t0: float,
|
||
windows: tuple[int, int, int],
|
||
*,
|
||
slow_ms: float,
|
||
sample_me: int,
|
||
seed: int,
|
||
collected_at: float,
|
||
) -> dict[str, Any]:
|
||
"""Вход `telemetry_lookup.py collect` (решение 9): id редких классов,
|
||
выборка `GET /users/me` чистого окна, окна."""
|
||
bounds = window_bounds(t0, windows)
|
||
me = [
|
||
r.request_id
|
||
for r in inside(src_reqs, bounds["clean"])
|
||
if r.cls == "me" and r.status == 200
|
||
]
|
||
random.Random(seed).shuffle(me)
|
||
return {
|
||
"run": {"start": bounds["run"][0], "end": bounds["run"][1]},
|
||
"clean": {"start": bounds["clean"][0], "end": bounds["clean"][1]},
|
||
"search": {"start": t0 - 60, "end": collected_at},
|
||
"classes": rare_ids(src_reqs, slow_ms),
|
||
"slow_ms": slow_ms,
|
||
"sample_me": me[:sample_me],
|
||
"routes": {
|
||
"read": ["GET", "/api/v1/chats/{chat_id}/messages"],
|
||
"write": ["POST", "/api/v1/chats/{chat_id}/messages"],
|
||
},
|
||
}
|
||
|
||
|
||
# ------------------------------------------------------ статистика
|
||
|
||
|
||
def binom_cdf(k: int, n: int, p: float) -> float:
|
||
if p <= 0:
|
||
return 1.0
|
||
if p >= 1:
|
||
return 1.0 if k >= n else 0.0
|
||
return sum(
|
||
math.exp(
|
||
math.lgamma(n + 1)
|
||
- math.lgamma(i + 1)
|
||
- math.lgamma(n - i + 1)
|
||
+ i * math.log(p)
|
||
+ (n - i) * math.log1p(-p),
|
||
)
|
||
for i in range(k + 1)
|
||
)
|
||
|
||
|
||
def binom_interval(
|
||
k: int,
|
||
n: int,
|
||
alpha: float = Z_ALPHA,
|
||
) -> tuple[float, float]:
|
||
"""Точный (Клоппер — Пирсон) интервал доли k/n уровня 1 − alpha."""
|
||
if n <= 0:
|
||
return (0.0, 1.0)
|
||
|
||
def solve(f: Any) -> float:
|
||
lo, hi = 0.0, 1.0
|
||
for _ in range(60):
|
||
mid = (lo + hi) / 2
|
||
if f(mid):
|
||
lo = mid
|
||
else:
|
||
hi = mid
|
||
return (lo + hi) / 2
|
||
|
||
lower = (
|
||
0.0
|
||
if k == 0
|
||
else solve(lambda p: 1 - binom_cdf(k - 1, n, p) < alpha / 2)
|
||
)
|
||
upper = 1.0 if k == n else solve(lambda p: binom_cdf(k, n, p) > alpha / 2)
|
||
return (lower, upper)
|
||
|
||
|
||
def values(rows: Iterable[Mapping[str, Any]], key: str) -> list[float]:
|
||
return [v for v in (num(r.get(key)) for r in rows) if v is not None]
|
||
|
||
|
||
def med(rows: Iterable[Mapping[str, Any]], key: str) -> float | None:
|
||
found = values(rows, key)
|
||
return statistics.median(found) if found else None
|
||
|
||
|
||
def by_variant(
|
||
rows: Iterable[Mapping[str, Any]],
|
||
) -> dict[str, list[Mapping[str, Any]]]:
|
||
out: dict[str, list[Mapping[str, Any]]] = {}
|
||
for row in rows:
|
||
out.setdefault(row["variant"], []).append(row)
|
||
return out
|
||
|
||
|
||
def noise(groups: Mapping[str, list[Mapping[str, Any]]], key: str) -> float:
|
||
"""Шум величины — |медиана full − медиана full-aa| (решение 12)."""
|
||
a, b = (
|
||
med(groups.get("full", []), key),
|
||
med(groups.get("full-aa", []), key),
|
||
)
|
||
if a is None or b is None:
|
||
return 0.0
|
||
return abs(a - b)
|
||
|
||
|
||
def effect(
|
||
groups: Mapping[str, list[Mapping[str, Any]]],
|
||
variant: str,
|
||
reference: str,
|
||
key: str,
|
||
) -> bool | None:
|
||
"""Эффект: разбросы повторов не пересекаются и |Δ медианы| больше шума."""
|
||
a, b = (
|
||
values(groups.get(variant, []), key),
|
||
values(
|
||
groups.get(reference, []),
|
||
key,
|
||
),
|
||
)
|
||
if not a or not b:
|
||
return None
|
||
apart = max(a) < min(b) or max(b) < min(a)
|
||
delta = abs(statistics.median(a) - statistics.median(b))
|
||
return apart and delta > noise(groups, key)
|
||
|
||
|
||
class Verdict:
|
||
"""Условия гипотезы или критерия: «нет данных», если нечем проверить
|
||
условие; «нет», если нарушено (причина названа)."""
|
||
|
||
def __init__(self, name: str) -> None:
|
||
self.name = name
|
||
self.failed: list[str] = []
|
||
self.missing: list[str] = []
|
||
self.notes: list[str] = []
|
||
|
||
def need(self, label: str, ok: bool | None, detail: str = "") -> None:
|
||
text = f"{label}{': ' + detail if detail else ''}"
|
||
if ok is None:
|
||
self.missing.append(text)
|
||
elif ok:
|
||
self.notes.append(text)
|
||
else:
|
||
self.failed.append(text)
|
||
|
||
@property
|
||
def result(self) -> str:
|
||
if self.failed:
|
||
return "нет"
|
||
if self.missing:
|
||
return "нет данных"
|
||
return "подтверждено"
|
||
|
||
def render(self) -> str:
|
||
parts = [f"**{self.name}: {self.result}.**"]
|
||
if self.failed:
|
||
parts.append("Нарушено: " + "; ".join(self.failed) + ".")
|
||
if self.missing:
|
||
parts.append("Нет данных: " + "; ".join(self.missing) + ".")
|
||
if self.notes:
|
||
parts.append("Выполнено: " + "; ".join(self.notes) + ".")
|
||
return " ".join(parts)
|
||
|
||
|
||
def fmt(value: float | None, digits: int = 2) -> str:
|
||
return "—" if value is None else f"{value:.{digits}f}"
|
||
|
||
|
||
def ratio(a: float | None, b: float | None) -> float | None:
|
||
return None if a is None or not b else a / b
|
||
|
||
|
||
def class_share(
|
||
classes: list[Mapping[str, Any]],
|
||
variant: str,
|
||
name: str,
|
||
field: str = "traces",
|
||
total_field: str = "client",
|
||
) -> tuple[int, int] | None:
|
||
"""Сумма найденного и всего (по умолчанию — счёт клиента) класса по
|
||
повторам варианта."""
|
||
rows = [
|
||
r for r in classes if r["variant"] == variant and r["class"] == name
|
||
]
|
||
if not rows or any(
|
||
num(r.get(key)) is None for r in rows for key in (field, total_field)
|
||
):
|
||
return None
|
||
found = int(sum(zero(r.get(field)) for r in rows))
|
||
total = int(sum(zero(r.get(total_field)) for r in rows))
|
||
return found, total
|
||
|
||
|
||
def kept(
|
||
classes: list[Mapping[str, Any]],
|
||
variant: str,
|
||
name: str,
|
||
) -> tuple[int, int] | None:
|
||
"""Сохранённые трассы класса, который критерии требуют полным: у
|
||
медленных записей — среди медленных по серверу, у неудачных загрузок —
|
||
цельные цепочки."""
|
||
if name == "write_slow":
|
||
return class_share(
|
||
classes,
|
||
variant,
|
||
name,
|
||
"server_slow_traces",
|
||
"server_slow",
|
||
)
|
||
if name == "upload_bad":
|
||
return class_share(classes, variant, name, "chains")
|
||
return class_share(classes, variant, name)
|
||
|
||
|
||
def log_share(
|
||
classes: list[Mapping[str, Any]],
|
||
variant: str,
|
||
name: str,
|
||
*,
|
||
server_slow: bool = False,
|
||
) -> tuple[int, int] | None:
|
||
"""Записи журнала класса, у которых `trace_id` ведёт на трассу в Tempo,
|
||
из записей с `trace_id`; `server_slow` — у медленных записей только
|
||
медленные по серверу (их правило выбора обязано сохранить)."""
|
||
if server_slow and name == "write_slow":
|
||
return class_share(
|
||
classes,
|
||
variant,
|
||
name,
|
||
"server_slow_linked",
|
||
"server_slow",
|
||
)
|
||
return class_share(
|
||
classes,
|
||
variant,
|
||
name,
|
||
"log_to_trace",
|
||
"logs_with_trace",
|
||
)
|
||
|
||
|
||
def quantile_error(rows: list[Mapping[str, Any]]) -> dict[str, float | None]:
|
||
"""Медиана по повторам относительной ошибки p95 и p99 из buckets против
|
||
журнала, по маршрутам (ошибка считается в каждом прогоне, а не по
|
||
медианам квантилей)."""
|
||
out = {}
|
||
for route in ("read", "write"):
|
||
for q in ("p95", "p99"):
|
||
errors = []
|
||
for row in rows:
|
||
b = num(row.get(f"q_{route}_{q}_buckets"))
|
||
log = num(row.get(f"q_{route}_{q}_log"))
|
||
if b is not None and log:
|
||
errors.append(abs(b - log) / log)
|
||
out[f"{route}_{q}"] = statistics.median(errors) if errors else None
|
||
return out
|
||
|
||
|
||
def verdicts(
|
||
rows: list[Mapping[str, Any]],
|
||
classes: list[Mapping[str, Any]],
|
||
collector_memory_mib: float,
|
||
) -> list[Verdict]:
|
||
g = by_variant(rows)
|
||
out = []
|
||
cpu = "replica_cpu_ms_per_req"
|
||
|
||
def m(variant: str, key: str) -> float | None:
|
||
return med(g.get(variant, []), key)
|
||
|
||
def eff(variant: str, reference: str, key: str) -> bool | None:
|
||
return effect(g, variant, reference, key)
|
||
|
||
# H1
|
||
h = Verdict("H1")
|
||
full_cpu, off_cpu = m("full", cpu), m("off", cpu)
|
||
r = ratio(full_cpu, off_cpu)
|
||
h.need(
|
||
"CPU реплик на запрос ≥ +10 %",
|
||
None if r is None else bool(eff("full", "off", cpu)) and r >= 1.10,
|
||
f"{fmt(full_cpu, 3)} против {fmt(off_cpu, 3)} мс",
|
||
)
|
||
p50 = (m("full", "light_p50"), m("off", "light_p50"))
|
||
p99 = (m("full", "light_p99"), m("off", "light_p99"))
|
||
d50 = None if None in p50 else p50[0] - p50[1]
|
||
d99 = None if None in p99 else p99[0] - p99[1]
|
||
h.need(
|
||
"p50 лёгких не больше +1 мс",
|
||
None if d50 is None else d50 <= 1,
|
||
f"Δ {fmt(d50)} мс",
|
||
)
|
||
rel50, rel99 = ratio(*p50), ratio(*p99)
|
||
h.need(
|
||
"p99 лёгких ≥ +2 мс и растёт сильнее p50",
|
||
None
|
||
if d99 is None or rel50 is None or rel99 is None
|
||
else d99 >= 2 and rel99 > rel50,
|
||
f"Δ {fmt(d99)} мс, рост p99 ×{fmt(rel99)} против p50 ×{fmt(rel50)}",
|
||
)
|
||
out.append(h)
|
||
# H2
|
||
h = Verdict("H2")
|
||
head_cpu = m("head", cpu)
|
||
h.need(
|
||
"надбавка CPU head ≤ половины надбавки full",
|
||
None
|
||
if None in (head_cpu, off_cpu, full_cpu)
|
||
else head_cpu - off_cpu <= 0.5 * (full_cpu - off_cpu),
|
||
f"head {fmt(head_cpu, 3)}, off {fmt(off_cpu, 3)}, full "
|
||
f"{fmt(full_cpu, 3)} мс",
|
||
)
|
||
tr = ratio(m("head", "tempo_spans_per_s"), m("full", "tempo_spans_per_s"))
|
||
h.need(
|
||
"приём Tempo ≤ 15 % full",
|
||
None if tr is None else tr <= 0.15,
|
||
f"{fmt(None if tr is None else tr * 100, 1)} %",
|
||
)
|
||
head_ratio = 0.1
|
||
for name in HEAD_CLASSES:
|
||
share = class_share(classes, "head", name)
|
||
if share is None or not share[1]:
|
||
h.need(f"доля трасс {name} около 0,1", None)
|
||
continue
|
||
lo, hi = binom_interval(*share)
|
||
h.need(
|
||
f"доля трасс {name} около 0,1",
|
||
lo <= head_ratio <= hi,
|
||
f"{share[0]}/{share[1]}, интервал {lo:.3f}–{hi:.3f}",
|
||
)
|
||
for name in HEAD_CLASSES:
|
||
logs = class_share(classes, "head", name, "logs")
|
||
h.need(
|
||
f"записи журнала {name} — 100 %",
|
||
None if logs is None else logs[0] == logs[1],
|
||
"" if logs is None else f"{logs[0]}/{logs[1]}",
|
||
)
|
||
# счёт в метрике — с начала процессов: у регистраций ещё пользователи
|
||
# prepare.js, поэтому «не меньше клиента»; медленные записи server span
|
||
# и клиент меряют по-разному — не сравниваются
|
||
for name in ("broken", "register", "upload_bad"):
|
||
metric = class_share(classes, "head", name, "metric")
|
||
h.need(
|
||
f"счёт {name} в метрике — 100 %",
|
||
None if metric is None else metric[0] >= metric[1],
|
||
"" if metric is None else f"{metric[0]}/{metric[1]}",
|
||
)
|
||
out.append(h)
|
||
# H3
|
||
h = Verdict("H3")
|
||
tail_effect = eff("tail", "full", cpu)
|
||
h.need(
|
||
"CPU реплик на запрос в пределах шума от full",
|
||
None if tail_effect is None else not tail_effect,
|
||
f"tail {fmt(m('tail', cpu), 3)}, full {fmt(full_cpu, 3)} мс",
|
||
)
|
||
peak = (
|
||
m("tail", "otel_collector_peak_mib"),
|
||
m("full", "otel_collector_peak_mib"),
|
||
)
|
||
h.need(
|
||
"пик памяти Collector ≥ full + 15 МиБ",
|
||
None if None in peak else peak[0] >= peak[1] + 15,
|
||
f"{fmt(peak[0], 1)} против {fmt(peak[1], 1)} МиБ",
|
||
)
|
||
ccpu = (m("tail", "otel_collector_cpu"), m("full", "otel_collector_cpu"))
|
||
h.need(
|
||
"CPU Collector не выше full больше чем на 20 %",
|
||
None if None in ccpu else ccpu[0] <= ccpu[1] * 1.2,
|
||
f"{fmt(ccpu[0], 3)} против {fmt(ccpu[1], 3)} ядра",
|
||
)
|
||
tr = ratio(m("tail", "tempo_spans_per_s"), m("full", "tempo_spans_per_s"))
|
||
h.need(
|
||
"приём Tempo ≤ 30 % full",
|
||
None if tr is None else tr <= 0.30,
|
||
f"{fmt(None if tr is None else tr * 100, 1)} %",
|
||
)
|
||
keep_all(h, classes, "tail", require_chain_bad=True)
|
||
chains = class_share(classes, "tail", "upload", "chains")
|
||
h.need(
|
||
"цепочки обычных загрузок ≥ 90 %",
|
||
None
|
||
if chains is None or not chains[1]
|
||
else chains[0] / chains[1] >= 0.9,
|
||
"" if chains is None else f"{chains[0]}/{chains[1]}",
|
||
)
|
||
refused = sum(values(g.get("tail", []), "col_span_lost")) + sum(
|
||
values(g.get("tail", []), "tail_dropped_too_early"),
|
||
)
|
||
h.need(
|
||
"нет отказов memory_limiter и dropped_too_early",
|
||
None if not g.get("tail") else refused == 0,
|
||
f"{refused:g}",
|
||
)
|
||
out.append(h)
|
||
# H4
|
||
h = Verdict("H4")
|
||
sr = ratio(m("metrics", "prom_series"), m("full", "prom_series"))
|
||
h.need(
|
||
"рядов Prometheus ≤ 40 % full",
|
||
None if sr is None else sr <= 0.4,
|
||
f"{fmt(None if sr is None else sr * 100, 1)} %",
|
||
)
|
||
ar = ratio(
|
||
m("metrics", "prom_samples_per_s"),
|
||
m("full", "prom_samples_per_s"),
|
||
)
|
||
h.need(
|
||
"отсчётов в секунду ≤ 40 % full",
|
||
None if ar is None else ar <= 0.4,
|
||
f"{fmt(None if ar is None else ar * 100, 1)} %",
|
||
)
|
||
for key, label in (
|
||
(cpu, "CPU реплик"),
|
||
("otel_collector_cpu", "CPU Collector"),
|
||
):
|
||
e = eff("metrics", "full", key)
|
||
h.need(f"{label} в пределах шума", None if e is None else not e)
|
||
coarse = quantile_error(g.get("metrics", []))
|
||
fine = quantile_error(g.get("full", []))
|
||
pairs = [
|
||
(coarse[k], fine[k])
|
||
for k in coarse
|
||
if coarse[k] is not None and fine[k] is not None
|
||
]
|
||
h.need(
|
||
"ошибка квантиля coarse ≥ 2 × default хотя бы на одном маршруте",
|
||
None if not pairs else any(c >= 2 * max(f, 1e-9) for c, f in pairs),
|
||
", ".join(
|
||
f"{k} {fmt(coarse[k], 3)} / {fmt(fine[k], 3)}" for k in coarse
|
||
),
|
||
)
|
||
out.append(h)
|
||
# H5
|
||
h = Verdict("H5")
|
||
# full — все редкие классы; tail и balanced — классы, которые их правила
|
||
# сохраняют (медленные записи — медленные по серверу)
|
||
for variant, names in (
|
||
("full", HEAD_CLASSES),
|
||
("tail", MUST_KEEP),
|
||
("balanced", MUST_KEEP),
|
||
):
|
||
for name in names:
|
||
share = log_share(
|
||
classes,
|
||
variant,
|
||
name,
|
||
server_slow=variant != "full",
|
||
)
|
||
if share is None or not share[1]:
|
||
h.need(f"{variant}: журнал → трасса {name} — 100 %", None)
|
||
continue
|
||
h.need(
|
||
f"{variant}: журнал → трасса {name} — 100 %",
|
||
share[0] == share[1],
|
||
f"{share[0]}/{share[1]}",
|
||
)
|
||
for name in HEAD_CLASSES:
|
||
share = log_share(classes, "head", name)
|
||
if share is None or not share[1]:
|
||
h.need(f"head: журнал → трасса {name} около 0,1", None)
|
||
continue
|
||
lo, hi = binom_interval(*share)
|
||
h.need(
|
||
f"head: журнал → трасса {name} около 0,1",
|
||
lo <= head_ratio <= hi,
|
||
f"{share[0]}/{share[1]}, интервал {lo:.3f}–{hi:.3f}",
|
||
)
|
||
out.append(h)
|
||
# H6
|
||
h = Verdict("H6")
|
||
telemetry_rows = [r for r in rows if r["variant"] != "off"]
|
||
lost = sum(sum(values(telemetry_rows, key)) for key in LOSS_KEYS)
|
||
h.need(
|
||
"потерь SDK и Collector и dropped_too_early нет",
|
||
None if not telemetry_rows else lost == 0,
|
||
f"{lost:g}",
|
||
)
|
||
gaps = values(telemetry_rows, "spans_gap")
|
||
h.need(
|
||
"span SDK → Collector → Tempo сходятся с точностью 1 % (выборка "
|
||
"отдельно)",
|
||
None if not gaps else max(gaps) <= 0.01,
|
||
f"наибольшее расхождение {fmt(max(gaps) * 100 if gaps else None)} %",
|
||
)
|
||
out.append(h)
|
||
# B1–B5 и H7
|
||
bs = criteria(g, classes, collector_memory_mib)
|
||
h = Verdict("H7")
|
||
for b in bs:
|
||
h.need(
|
||
f"{b.name} — {b.result}",
|
||
None if b.result == "нет данных" else b.result == "подтверждено",
|
||
)
|
||
out.append(h)
|
||
# H8
|
||
h = Verdict("H8")
|
||
t, lk, pr = (
|
||
m("full", k) for k in ("tempo_cpu", "loki_cpu", "prometheus_cpu")
|
||
)
|
||
h.need(
|
||
"у full CPU Tempo больше Loki и Prometheus вместе",
|
||
None if None in (t, lk, pr) else t > lk + pr,
|
||
f"{fmt(t, 3)} против {fmt(lk, 3)} + {fmt(pr, 3)}",
|
||
)
|
||
dt, dl, dp = (m("full", f"disk_{k}_bytes") for k in DISKS[::-1])
|
||
h.need(
|
||
"рост Tempo на диске больше Loki и Prometheus вместе",
|
||
None if None in (dt, dl, dp) else dt > dl + dp,
|
||
f"{fmt(dt, 0)} против {fmt(dl, 0)} + {fmt(dp, 0)} байт",
|
||
)
|
||
for variant in ("head", "tail", "balanced"):
|
||
v = m(variant, "tempo_cpu")
|
||
h.need(
|
||
f"CPU Tempo у {variant} ≤ половины full",
|
||
None if None in (v, t) else v <= 0.5 * t,
|
||
f"{fmt(v, 3)} против {fmt(t, 3)}",
|
||
)
|
||
out.append(h)
|
||
return [*out, *bs]
|
||
|
||
|
||
def keep_all(
|
||
h: Verdict,
|
||
classes: list[Mapping[str, Any]],
|
||
variant: str,
|
||
*,
|
||
require_chain_bad: bool,
|
||
) -> None:
|
||
labels = {
|
||
"broken": "трассы 500 API — 100 %",
|
||
"write_slow": "трассы медленных по серверу записей — 100 %",
|
||
"upload_bad": "цепочки неудачных загрузок — 100 %",
|
||
}
|
||
for name in MUST_KEEP:
|
||
if name == "upload_bad" and not require_chain_bad:
|
||
continue
|
||
share = kept(classes, variant, name)
|
||
h.need(
|
||
labels[name],
|
||
None if share is None else share[0] == share[1],
|
||
"" if share is None else f"{share[0]}/{share[1]}",
|
||
)
|
||
# регистрация — редкий быстрый маршрут: сохраняется с долей остальных,
|
||
# в вердикт не входит, показывается числом
|
||
share = class_share(classes, variant, "register")
|
||
if share is not None:
|
||
h.notes.append(
|
||
f"регистрации (редкий быстрый маршрут, справочно) {share[0]}/"
|
||
f"{share[1]}",
|
||
)
|
||
|
||
|
||
def criteria(
|
||
g: Mapping[str, list[Mapping[str, Any]]],
|
||
classes: list[Mapping[str, Any]],
|
||
collector_memory_mib: float,
|
||
) -> list[Verdict]:
|
||
"""Критерии сбалансированной конфигурации B1–B5 (спек `load-benchmark`,
|
||
«Цена телеметрии в отчёте»)."""
|
||
|
||
def m(variant: str, key: str) -> float | None:
|
||
return med(g.get(variant, []), key)
|
||
|
||
out = []
|
||
b = Verdict("B1")
|
||
keep_all(b, classes, "balanced", require_chain_bad=True)
|
||
out.append(b)
|
||
b = Verdict("B2")
|
||
tb, tf, tt = (
|
||
m(v, "tempo_spans_per_s") for v in ("balanced", "full", "tail")
|
||
)
|
||
b.need(
|
||
"приём Tempo ≤ 25 % full",
|
||
None if None in (tb, tf) or not tf else tb / tf <= 0.25,
|
||
f"{fmt(ratio(tb, tf) and ratio(tb, tf) * 100, 1)} %",
|
||
)
|
||
b.need(
|
||
"приём Tempo меньше, чем у tail",
|
||
None if None in (tb, tt) else tb < tt,
|
||
f"{fmt(tb, 1)} против {fmt(tt, 1)} span/с",
|
||
)
|
||
sr = ratio(m("balanced", "prom_series"), m("full", "prom_series"))
|
||
b.need(
|
||
"рядов Prometheus ≤ 40 % full",
|
||
None if sr is None else sr <= 0.4,
|
||
f"{fmt(None if sr is None else sr * 100, 1)} %",
|
||
)
|
||
out.append(b)
|
||
b = Verdict("B3")
|
||
rows = g.get("balanced", [])
|
||
lost = sum(sum(values(rows, key)) for key in LOSS_KEYS)
|
||
b.need(
|
||
"потерь и отброшенных до решения нет",
|
||
None if not rows else lost == 0,
|
||
f"{lost:g}",
|
||
)
|
||
peaks = values(rows, "otel_collector_peak_mib")
|
||
b.need(
|
||
"пик памяти Collector ≤ 2/3 лимита",
|
||
None if not peaks else max(peaks) <= collector_memory_mib * 2 / 3,
|
||
f"{fmt(max(peaks) if peaks else None, 1)} из "
|
||
f"{collector_memory_mib:g} МиБ",
|
||
)
|
||
out.append(b)
|
||
b = Verdict("B4")
|
||
cb, co = (
|
||
m("balanced", "replica_cpu_ms_per_req"),
|
||
m(
|
||
"off",
|
||
"replica_cpu_ms_per_req",
|
||
),
|
||
)
|
||
b.need(
|
||
"CPU реплик на запрос не больше +25 %",
|
||
None if None in (cb, co) or not co else cb / co <= 1.25,
|
||
f"{fmt(cb, 3)} против {fmt(co, 3)} мс",
|
||
)
|
||
for key, limit in (("light_p95", 3), ("light_p99", 10)):
|
||
a, z = m("balanced", key), m("off", key)
|
||
b.need(
|
||
f"{key} не больше +{limit} мс",
|
||
None if None in (a, z) else a - z <= limit,
|
||
f"Δ {fmt(None if None in (a, z) else a - z)} мс",
|
||
)
|
||
out.append(b)
|
||
b = Verdict("B5")
|
||
for name in MUST_KEEP:
|
||
share = log_share(classes, "balanced", name, server_slow=True)
|
||
b.need(
|
||
f"журнал → трасса {name}",
|
||
None if share is None else share[0] == share[1],
|
||
"" if share is None else f"{share[0]}/{share[1]}",
|
||
)
|
||
out.append(b)
|
||
return out
|
||
|
||
|
||
# ------------------------------------------------------------- сводка
|
||
|
||
|
||
ORDER = ("off", "full", "full-aa", "head", "tail", "metrics", "balanced")
|
||
|
||
|
||
def spread(rows: list[Mapping[str, Any]], key: str, digits: int = 2) -> str:
|
||
return reconcile.spread((r.get(key) for r in rows), digits)
|
||
|
||
|
||
def render_summary(
|
||
meta: Mapping[str, Any],
|
||
rows: list[Mapping[str, Any]],
|
||
classes: list[Mapping[str, Any]],
|
||
comps: list[Mapping[str, Any]],
|
||
) -> str:
|
||
g = by_variant(rows)
|
||
names = [n for n in ORDER if n in g]
|
||
profile = meta.get("profile", {})
|
||
info = meta.get("telemetry_cost", {})
|
||
collector_mib = (
|
||
reconcile_size(profile.get("collector_memory", "256m")) / 2**20
|
||
)
|
||
lines = [
|
||
f"# Цена телеметрии — метка {meta.get('label')}",
|
||
"",
|
||
f"Commit `{meta.get('git', {}).get('commit')}`, dirty "
|
||
f"{meta.get('git', {}).get('dirty')}; профиль {profile.get('name')}; "
|
||
f"окна {info.get('windows')} с; повторов {profile.get('repeats')}; "
|
||
f"статус {meta.get('status')}.",
|
||
"",
|
||
"Медиана и разброс (min–max) повторов; шум — |full − full-aa|.",
|
||
"",
|
||
"## Задержки лёгких запросов (чистое окно), мс",
|
||
"",
|
||
"| Вариант | p50 | p95 | p99 | ошибок |",
|
||
"|---|---|---|---|---|",
|
||
]
|
||
for name in names:
|
||
r = g[name]
|
||
lines.append(
|
||
f"| {name} | {spread(r, 'light_p50')} | {spread(r, 'light_p95')} "
|
||
f"| {spread(r, 'light_p99')} | {spread(r, 'light_errors', 0)} |",
|
||
)
|
||
lines += [
|
||
"",
|
||
"## Остальные классы (чистое окно): p95, мс",
|
||
"",
|
||
"| Вариант | " + " | ".join(CLASSES) + " |",
|
||
"|---|" + "---|" * len(CLASSES),
|
||
]
|
||
for name in names:
|
||
lines.append(
|
||
f"| {name} | "
|
||
+ " | ".join(spread(g[name], f"{c}_p95", 1) for c in CLASSES)
|
||
+ " |",
|
||
)
|
||
lines += [
|
||
"",
|
||
"## CPU и память",
|
||
"",
|
||
"| Вариант | CPU реплик на запрос, мс | реплики, ядра | приложение "
|
||
"| конвейер телеметрии | прочее | Collector пик, МиБ |",
|
||
"|---|---|---|---|---|---|---|",
|
||
]
|
||
for name in names:
|
||
r = g[name]
|
||
lines.append(
|
||
f"| {name} | {spread(r, 'replica_cpu_ms_per_req', 3)} | "
|
||
f"{spread(r, 'replica_cpu_cores', 3)} | "
|
||
f"{spread(r, 'app_cpu_cores', 3)} | "
|
||
f"{spread(r, 'telemetry_cpu_cores', 3)} | "
|
||
f"{spread(r, 'rest_cpu_cores', 3)} | "
|
||
f"{spread(r, 'otel_collector_peak_mib', 1)} |",
|
||
)
|
||
noise_cpu = noise(g, "replica_cpu_ms_per_req")
|
||
lines += [
|
||
"",
|
||
f"Шум CPU реплик на запрос (A/A): {noise_cpu:.4f} мс.",
|
||
"",
|
||
"### По компонентам: CPU, ядра (медиана), пик памяти, МиБ",
|
||
"",
|
||
"| Вариант | "
|
||
+ " | ".join(f"{role} CPU / пик" for role in TELEMETRY_ROLES)
|
||
+ " |",
|
||
"|---|" + "---|" * len(TELEMETRY_ROLES),
|
||
]
|
||
for name in names:
|
||
r = g[name]
|
||
cells = []
|
||
for role in TELEMETRY_ROLES:
|
||
p = role.replace("-", "_")
|
||
cells.append(
|
||
f"{fmt(med(r, f'{p}_cpu'), 3)} / "
|
||
f"{fmt(med(r, f'{p}_peak_mib'), 1)}",
|
||
)
|
||
lines.append(f"| {name} | " + " | ".join(cells) + " |")
|
||
lines += [
|
||
"",
|
||
"## Приём по сигналам (чистое окно, в секунду) и рост на диске",
|
||
"",
|
||
"| Вариант | span Collector | записей | точек | span Tempo | байт Tempo "
|
||
"| строк Loki | байт Loki | отсчётов Prometheus | рядов | диск "
|
||
"Prometheus / Loki / Tempo, МиБ |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for name in names:
|
||
r = g[name]
|
||
disk = " / ".join(
|
||
fmt(
|
||
None
|
||
if med(r, f"disk_{d}_bytes") is None
|
||
else med(r, f"disk_{d}_bytes") / 2**20,
|
||
1,
|
||
)
|
||
for d in DISKS
|
||
)
|
||
lines.append(
|
||
f"| {name} | {spread(r, 'col_accepted_spans_per_s', 0)} | "
|
||
f"{spread(r, 'col_accepted_logs_per_s', 0)} | "
|
||
f"{spread(r, 'col_accepted_points_per_s', 0)} | "
|
||
f"{spread(r, 'tempo_spans_per_s', 0)} | "
|
||
f"{spread(r, 'tempo_bytes_per_s', 0)} | "
|
||
f"{spread(r, 'loki_lines_per_s', 0)} | "
|
||
f"{spread(r, 'loki_bytes_per_s', 0)} | "
|
||
f"{spread(r, 'prom_samples_per_s', 0)} | "
|
||
f"{spread(r, 'prom_series', 0)} | {disk} |",
|
||
)
|
||
run_s = sum(info.get("windows") or [0])
|
||
lines += [
|
||
"",
|
||
"Пересчёт на сутки — приём в секунду × 86 400; срок хранения 7 суток "
|
||
"— × 604 800 (рост на диске за прогон — журнал предзаписи, без "
|
||
f"сжатых блоков: прогон {run_s} с).",
|
||
"",
|
||
"## Потери по звеньям и отброшенное выборкой (весь прогон)",
|
||
"",
|
||
"| Вариант | span SDK выбрано / отброшено в голове | SDK отправил | "
|
||
"Collector принял / отправил | решения tail: сохр. / нет | Tempo "
|
||
"принял | потери SDK span / записей | потери Collector span / "
|
||
"записей | dropped_too_early | расхождение |",
|
||
"|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for name in names:
|
||
r = g[name]
|
||
lines.append(
|
||
f"| {name} | {spread(r, 'sdk_spans_sampled', 0)} / "
|
||
f"{spread(r, 'sdk_spans_dropped', 0)} | "
|
||
f"{spread(r, 'sdk_spans_exported', 0)} | "
|
||
f"{spread(r, 'col_spans_accepted', 0)} / "
|
||
f"{spread(r, 'col_spans_sent', 0)} | "
|
||
f"{spread(r, 'tail_sampled', 0)} / "
|
||
f"{spread(r, 'tail_not_sampled', 0)} | "
|
||
f"{spread(r, 'tempo_spans_total', 0)} | "
|
||
f"{spread(r, 'sdk_span_lost', 0)} / {spread(r, 'sdk_log_lost', 0)} "
|
||
f"| {spread(r, 'col_span_lost', 0)} / "
|
||
f"{spread(r, 'col_log_lost', 0)} | "
|
||
f"{spread(r, 'tail_dropped_too_early', 0)} | "
|
||
f"{spread(r, 'spans_gap', 4)} |",
|
||
)
|
||
lines += [
|
||
"",
|
||
"## Время до ready и итоги заданий",
|
||
"",
|
||
"| Вариант | p50, с | p95, с | принято / ready | bad принято / "
|
||
"failed | медленных записей | недосланных |",
|
||
"|---|---|---|---|---|---|---|",
|
||
]
|
||
for name in names:
|
||
r = g[name]
|
||
lines.append(
|
||
f"| {name} | {spread(r, 'ttr_p50_s', 3)} | "
|
||
f"{spread(r, 'ttr_p95_s', 3)} | "
|
||
f"{int(sum(values(r, 'accepted')))} / "
|
||
f"{int(sum(values(r, 'ready')))} | "
|
||
f"{int(sum(values(r, 'bad_accepted')))} / "
|
||
f"{int(sum(values(r, 'bad_failed')))} | "
|
||
f"{int(sum(values(r, 'slow_writes')))} | "
|
||
f"{int(sum(values(r, 'dropped')))} |",
|
||
)
|
||
lost_runs = [r["run"] for r in rows if "lost" in (r.get("flags") or "")]
|
||
if lost_runs:
|
||
lines += ["", f"Принятая работа не дошла до итога: {lost_runs}."]
|
||
lines += [
|
||
"",
|
||
"## Что сохраняет каждый вариант (сумма повторов)",
|
||
"",
|
||
"Трасса по `X-Request-Id`, цепочка загрузки (server span API, "
|
||
"producer relay, consumer worker с итогом), запись журнала, переход "
|
||
"журнал → трасса, счёт в метрике (конец прогона).",
|
||
"",
|
||
"| Вариант | Класс | по клиенту | трасс | цепочек | записей | с "
|
||
"trace_id | журнал → трасса | метрика | медленных по серверу / их "
|
||
"трасс |",
|
||
"|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for name in names:
|
||
for cls in RARE:
|
||
rows_c = [
|
||
r
|
||
for r in classes
|
||
if r["variant"] == name and r["class"] == cls
|
||
]
|
||
if not rows_c:
|
||
continue
|
||
|
||
def total(
|
||
key: str,
|
||
items: list[Mapping[str, Any]] = rows_c,
|
||
) -> str:
|
||
found = values(items, key)
|
||
return "—" if not found else f"{int(sum(found))}"
|
||
|
||
lines.append(
|
||
f"| {name} | {cls} | {total('client')} | {total('traces')} | "
|
||
f"{total('chains')} | {total('logs')} | "
|
||
f"{total('logs_with_trace')} | {total('log_to_trace')} | "
|
||
f"{total('metric')} | {total('server_slow')} / "
|
||
f"{total('server_slow_traces')} |",
|
||
)
|
||
lines += [
|
||
"",
|
||
"## Точность квантилей: из buckets против журнала (чистое окно), с",
|
||
"",
|
||
"| Вариант | read p95 | read p99 | write p95 | write p99 |",
|
||
"|---|---|---|---|---|",
|
||
]
|
||
for name in names:
|
||
r = g[name]
|
||
cells = []
|
||
for route in ("read", "write"):
|
||
for q in ("p95", "p99"):
|
||
cells.append(
|
||
f"{fmt(med(r, f'q_{route}_{q}_buckets'), 4)} / "
|
||
f"{fmt(med(r, f'q_{route}_{q}_log'), 4)}",
|
||
)
|
||
lines.append(f"| {name} | " + " | ".join(cells) + " |")
|
||
lines += ["", "## Гипотезы и критерии", ""]
|
||
for verdict in verdicts(rows, classes, collector_mib):
|
||
lines += [verdict.render(), ""]
|
||
flagged = [f"{r['run']}: {r['flags']}" for r in rows if r.get("flags")]
|
||
lines += ["## Пометки прогонов", ""]
|
||
lines += [f"- {item}" for item in flagged] or ["нет"]
|
||
warnings = meta.get("host_warnings") or []
|
||
if warnings:
|
||
lines += ["", "## Предупреждения о хосте", ""]
|
||
lines += [f"- {w}" for w in warnings]
|
||
lines += [
|
||
"",
|
||
"Графики: `cpu-by-group.svg`, `latency-light.svg`, `ingest.svg`, "
|
||
"`traces-by-class.svg`.",
|
||
"",
|
||
]
|
||
del comps
|
||
return "\n".join(lines)
|
||
|
||
|
||
def reconcile_size(text: str) -> int:
|
||
units = {"k": 2**10, "m": 2**20, "g": 2**30}
|
||
text = str(text).strip().lower()
|
||
if text and text[-1] in units:
|
||
return int(float(text[:-1]) * units[text[-1]])
|
||
return int(float(text))
|
||
|
||
|
||
def grouped_bars(
|
||
title: str,
|
||
unit: str,
|
||
groups: list[str],
|
||
series: list[tuple[str, list[float | None]]],
|
||
*,
|
||
stacked: bool = False,
|
||
) -> str:
|
||
"""Столбики по вариантам: рядом (или стопкой) несколько рядов."""
|
||
width, height = 860, 420
|
||
left, right, top, bottom = 70, 20, 60, 70
|
||
svg = plots.Svg(width, height, title)
|
||
totals = [
|
||
sum(v or 0 for _, vals in series for v in [vals[i]])
|
||
if stacked
|
||
else max((vals[i] or 0) for _, vals in series)
|
||
for i in range(len(groups))
|
||
]
|
||
high = max(totals + [0.0]) or 1.0
|
||
ticks = plots.nice_ticks(high)
|
||
top_value = ticks[-1]
|
||
plot_h = height - top - bottom
|
||
plot_w = width - left - right
|
||
|
||
def y(value: float) -> float:
|
||
return top + plot_h * (1 - value / top_value)
|
||
|
||
for tick in ticks:
|
||
svg.line(left, y(tick), width - right, y(tick))
|
||
svg.text(left - 6, y(tick) + 4, f"{tick:g}", anchor="end", size=11)
|
||
svg.text(18, top + plot_h / 2, unit, rotate=True, anchor="middle", size=11)
|
||
slot = plot_w / max(1, len(groups))
|
||
bars = 1 if stacked else max(1, len(series))
|
||
bar_w = min(40.0, slot * 0.7 / bars)
|
||
for i, group in enumerate(groups):
|
||
x0 = left + slot * i + (slot - bar_w * bars) / 2
|
||
base = 0.0
|
||
for j, (label, vals) in enumerate(series):
|
||
value = vals[i]
|
||
if value is None:
|
||
continue
|
||
color = plots.SERIES[j % len(plots.SERIES)]
|
||
if stacked:
|
||
svg.bar(
|
||
x0,
|
||
y(base + value),
|
||
bar_w,
|
||
y(base) - y(base + value),
|
||
color,
|
||
f"{group} {label}: {value:.3g}",
|
||
top=j == len(series) - 1,
|
||
)
|
||
base += value
|
||
else:
|
||
x = x0 + j * bar_w
|
||
svg.bar(
|
||
x,
|
||
y(value),
|
||
bar_w - 2,
|
||
y(0) - y(value),
|
||
color,
|
||
f"{group} {label}: {value:.3g}",
|
||
top=True,
|
||
)
|
||
svg.text(
|
||
left + slot * i + slot / 2,
|
||
height - bottom + 18,
|
||
group,
|
||
anchor="middle",
|
||
size=11,
|
||
)
|
||
svg.legend(
|
||
[
|
||
(plots.SERIES[j % len(plots.SERIES)], label, "box")
|
||
for j, (label, _) in enumerate(series)
|
||
],
|
||
left,
|
||
height - 24,
|
||
plot_w,
|
||
)
|
||
return svg.render()
|
||
|
||
|
||
def render_plots(
|
||
rows: list[Mapping[str, Any]],
|
||
classes: list[Mapping[str, Any]],
|
||
) -> dict[str, str]:
|
||
g = by_variant(rows)
|
||
names = [n for n in ORDER if n in g]
|
||
|
||
def series(keys: list[tuple[str, str]]) -> list[tuple[str, list[Any]]]:
|
||
return [
|
||
(label, [med(g[n], key) for n in names]) for key, label in keys
|
||
]
|
||
|
||
out = {
|
||
"cpu-by-group.svg": grouped_bars(
|
||
"CPU по группам в чистом окне",
|
||
"ядра",
|
||
names,
|
||
series(
|
||
[
|
||
("app_cpu_cores", "приложение"),
|
||
("telemetry_cpu_cores", "конвейер телеметрии"),
|
||
("rest_cpu_cores", "прочее"),
|
||
],
|
||
),
|
||
stacked=True,
|
||
),
|
||
"latency-light.svg": grouped_bars(
|
||
"Лёгкие запросы: p50, p95, p99",
|
||
"мс",
|
||
names,
|
||
series(
|
||
[
|
||
("light_p50", "p50"),
|
||
("light_p95", "p95"),
|
||
("light_p99", "p99"),
|
||
],
|
||
),
|
||
),
|
||
}
|
||
full = g.get("full", [])
|
||
|
||
def relative(key: str) -> list[float | None]:
|
||
base = med(full, key)
|
||
return [
|
||
None
|
||
if med(g[n], key) is None or not base
|
||
else med(g[n], key) / base * 100
|
||
for n in names
|
||
]
|
||
|
||
out["ingest.svg"] = grouped_bars(
|
||
"Приём по сигналам, % от full",
|
||
"%",
|
||
names,
|
||
[
|
||
("span Tempo", relative("tempo_spans_per_s")),
|
||
("строк Loki", relative("loki_lines_per_s")),
|
||
("отсчётов Prometheus", relative("prom_samples_per_s")),
|
||
],
|
||
)
|
||
shares = []
|
||
for cls in RARE:
|
||
vals = []
|
||
for n in names:
|
||
share = class_share(classes, n, cls)
|
||
vals.append(
|
||
None
|
||
if share is None or not share[1]
|
||
else share[0] / share[1] * 100,
|
||
)
|
||
shares.append((cls, vals))
|
||
out["traces-by-class.svg"] = grouped_bars(
|
||
"Сохранённые трассы редких событий, %",
|
||
"%",
|
||
names,
|
||
shares,
|
||
)
|
||
return out
|
||
|
||
|
||
def read_csv(path: Path) -> list[dict[str, Any]]:
|
||
return overload.read_csv(path)
|
||
|
||
|
||
# ------------------------------------------------------------ selftest
|
||
|
||
|
||
def selftest() -> None:
|
||
table = variants(0.1, 30)
|
||
assert set(table) == set(ORDER)
|
||
assert sdk_env(table["off"]) == {}
|
||
assert sdk_env(table["head"])["TELEMETRY__TRACE_SAMPLE_RATIO"] == "0.1"
|
||
assert sdk_env(table["metrics"])["TELEMETRY__METRIC_BUCKETS"] == "coarse"
|
||
assert wait_after(table["full"]) == 35
|
||
assert wait_after(table["metrics"]) == 50
|
||
assert wait_after(table["tail"]) == 52
|
||
assert wait_after(table["balanced"]) == 35
|
||
assert wait_after(table["off"]) == 0
|
||
compose = json.loads(variant_compose(table["balanced"], ROOT))
|
||
assert compose["services"]["otel-collector"]["command"][1].endswith(
|
||
"layer.yaml",
|
||
)
|
||
assert compose["services"]["prometheus"]["volumes"][0]["target"] == (
|
||
PROMETHEUS_TARGET
|
||
)
|
||
assert "services" in json.loads(variant_compose(table["off"], ROOT))
|
||
|
||
plan = plan_runs(ORDER, 3, 20261114)
|
||
assert len(plan) == 21
|
||
for a, b in zip(plan, plan[1:], strict=False):
|
||
assert a["variant"] != b["variant"], (a, b)
|
||
assert {p["variant"] for p in plan[:7]} == set(ORDER)
|
||
|
||
base = (ROOT / PROMETHEUS_CONFIG).read_text(encoding="utf-8")
|
||
lean = (ROOT / PROMETHEUS_LEAN).read_text(encoding="utf-8")
|
||
assert lean_problems(lean, base) == [], lean_problems(lean, base)
|
||
assert lean_problems(lean, base.replace("15s", "20s", 1))
|
||
assert lean_problems(lean.replace(LEAN_END, "", 1), base)
|
||
|
||
base_cfg = {
|
||
"processors": {"memory_limiter": {}, "batch": {}},
|
||
"service": {
|
||
"pipelines": {
|
||
"traces": {"processors": ["memory_limiter", "batch"]},
|
||
"logs": {"processors": ["memory_limiter", "batch"]},
|
||
},
|
||
},
|
||
}
|
||
good = json.loads(json.dumps(base_cfg))
|
||
good["processors"]["tail_sampling"] = {"policies": []}
|
||
good["service"]["pipelines"]["traces"]["processors"] = [
|
||
"memory_limiter",
|
||
"tail_sampling",
|
||
"batch",
|
||
]
|
||
assert collector_layer_problems(base_cfg, good) == []
|
||
bad = json.loads(json.dumps(good))
|
||
bad["service"]["pipelines"]["logs"]["processors"].append("extra")
|
||
assert any("logs" in p for p in collector_layer_problems(base_cfg, bad))
|
||
assert collector_layer_problems(base_cfg, base_cfg)
|
||
|
||
records = {
|
||
"api:a": [
|
||
{
|
||
"event": "Started API",
|
||
"telemetry_enabled": True,
|
||
"telemetry_trace_sample_ratio": 0.1,
|
||
"telemetry_metric_buckets": "default",
|
||
"telemetry_metric_export_interval_seconds": 15,
|
||
},
|
||
],
|
||
"worker": [{"event": "Started worker", "telemetry_enabled": False}],
|
||
}
|
||
found = startup_problems(records, table["head"])
|
||
assert found and all(p.startswith("worker:") for p in found), found
|
||
assert startup_problems({"relay": []}, table["off"])
|
||
assert (
|
||
layer_problems(
|
||
{"tail_sampling": True, "lean_prometheus": True},
|
||
table["balanced"],
|
||
)
|
||
== []
|
||
)
|
||
assert layer_problems({"tail_sampling": True}, table["full"])
|
||
|
||
t0_ms = 1_800_000_000_000
|
||
lines = [f"t-t0,{t0_ms}"]
|
||
reqs_spec = [
|
||
("read", 35_000, 5.0, 200),
|
||
("read", 36_000, 7.0, 200),
|
||
("write", 37_000, 9.0, 201),
|
||
("write", 125_000, 400.0, 201),
|
||
("broken", 50_000, 4.0, 500),
|
||
("register", 60_000, 500.0, 201),
|
||
("upload", 40_000, 80.0, 202),
|
||
("upload_bad", 41_000, 70.0, 202),
|
||
("me", 45_000, 3.0, 200),
|
||
("me", 46_000, 3.0, 401),
|
||
]
|
||
for n, (cls, offset, dur, status) in enumerate(reqs_spec):
|
||
aid = f"a{n}" if cls.startswith("upload") else "-"
|
||
lines.append(
|
||
f"t,{cls},{t0_ms + offset},{dur},{status},r{n:031x},inst{n % 2},"
|
||
f"{aid}",
|
||
)
|
||
lines.append("garbage")
|
||
t0, reqs = parse_k6("\n".join(lines))
|
||
assert t0 == t0_ms / 1000 and len(reqs) == len(reqs_spec)
|
||
bounds = window_bounds(t0, (30, 90, 30))
|
||
clean = inside(reqs, bounds["clean"])
|
||
assert len(clean) == 9, len(clean)
|
||
light = latency(clean, LIGHT)
|
||
assert light["n"] == 5 and light["errors"] == 1 and light["p99"] == 9.0
|
||
ids = rare_ids(reqs, 250)
|
||
assert len(ids["write_slow"]) == 1 and len(ids["broken"]) == 1
|
||
|
||
lo, hi = binom_interval(3, 30)
|
||
assert lo < 0.1 < hi, (lo, hi)
|
||
lo, hi = binom_interval(30, 30)
|
||
assert hi == 1.0 and lo > 0.88
|
||
lo, hi = binom_interval(0, 30)
|
||
assert lo == 0.0 and 0.1 < hi < 0.13, hi
|
||
|
||
assert parse_du("100\t/v/prometheus\n200 /v/tempo\nx") == {
|
||
"prometheus": 100,
|
||
"tempo": 200,
|
||
}
|
||
ns = int(t0 * 1e9)
|
||
sampler = []
|
||
for second in range(0, 160, 1):
|
||
t = ns + second * 10**9
|
||
sampler.append(
|
||
f"cg {t} - aaaaaaaaaaaa {second * 400_000} {second * 10} 0 0 "
|
||
f"{100 * 2**20} 0",
|
||
)
|
||
sampler.append(
|
||
f"cg {t} - cccccccccccc {second * 100_000} {second * 10} "
|
||
f"{second} 0 {50 * 2**20} 0",
|
||
)
|
||
sampler.append(f"vm {t} 10 20 0")
|
||
sampler.append(f"mp {t} cccccccccccc {80 * 2**20} {60 * 2**20} 0")
|
||
meta = overload.Meta(200, "ready", "-", 3, t0 + 41)
|
||
metas = {
|
||
"a6": meta,
|
||
"a7": overload.Meta(200, "failed", "invalid_image", 0, t0 + 42),
|
||
}
|
||
lookup = {
|
||
"classes": {
|
||
"broken": {
|
||
"traces": 1,
|
||
"logs": 1,
|
||
"logs_with_trace": 1,
|
||
"log_to_trace": 1,
|
||
},
|
||
},
|
||
"totals": {
|
||
"sdk_span_exported_ok": 1000,
|
||
"col_accepted_spans": 1000,
|
||
"col_sent_spans": 1000,
|
||
"tempo_spans": 995,
|
||
"sdk_span_processed_lost": 0,
|
||
},
|
||
"clean": {
|
||
"tempo_spans": {"delta": 900, "seconds": 90},
|
||
"prom_series": {"last": 5000},
|
||
},
|
||
"rare_metrics": {"broken": 1},
|
||
"quantiles": {"read": {"p95_buckets": 0.01, "p95_log": 0.007}},
|
||
"layers": {"tail_sampling": False, "lean_prometheus": False},
|
||
}
|
||
src = Sources(
|
||
run={"run": "full-r1", "variant": "full", "repeat": 1, "order": 1},
|
||
variant=table["full"],
|
||
windows=(30, 90, 30),
|
||
t0=t0,
|
||
reqs=reqs,
|
||
dropped=0,
|
||
sampler_text="\n".join(sampler),
|
||
containers={
|
||
"aaaaaaaaaaaa": {"name": "mtusi-bench-api-1", "role": "api"},
|
||
"cccccccccccc": {
|
||
"name": "mtusi-bench-otel-collector",
|
||
"role": "otel-collector",
|
||
},
|
||
},
|
||
metas=metas,
|
||
lookup=lookup,
|
||
du={"prometheus": 1, "loki": 2, "tempo": 3},
|
||
slow_ms=250,
|
||
)
|
||
comps = components(src)
|
||
assert len(comps) == 2
|
||
api = next(c for c in comps if c["role"] == "api")
|
||
assert abs(api["cpu_cores"] - 0.4) < 1e-6, api
|
||
col = next(c for c in comps if c["role"] == "otel-collector")
|
||
assert col["peak_mib"] == 80.0 and col["throttle"] == 0.1, col
|
||
row = run_row(src, comps)
|
||
assert set(row) == set(RUN_FIELDS), set(RUN_FIELDS) ^ set(row)
|
||
assert row["tempo_spans_per_s"] == 10.0
|
||
assert row["ready"] == 1 and row["bad_failed"] == 1
|
||
assert row["spans_gap"] == 0.005
|
||
assert "lost" not in row["flags"]
|
||
assert row["q_read_p95_buckets"] == 0.01
|
||
assert row["replica_cpu_ms_per_req"] is not None
|
||
crow = class_rows(src)
|
||
assert {r["class"] for r in crow} == set(RARE)
|
||
events = events_for_lookup(
|
||
reqs,
|
||
t0,
|
||
(30, 90, 30),
|
||
slow_ms=250,
|
||
sample_me=10,
|
||
seed=1,
|
||
collected_at=t0 + 300,
|
||
)
|
||
assert events["classes"]["write_slow"] and len(events["sample_me"]) == 1
|
||
|
||
rows = []
|
||
classes = []
|
||
rng = random.Random(7)
|
||
for variant in ORDER:
|
||
for repeat in (1, 2, 3):
|
||
base_cpu = {"off": 1.0, "head": 1.05}.get(variant, 1.2)
|
||
rows.append(
|
||
{
|
||
"run": f"{variant}-r{repeat}",
|
||
"variant": variant,
|
||
"repeat": repeat,
|
||
"replica_cpu_ms_per_req": base_cpu + rng.random() * 0.01,
|
||
"light_p50": 3.0,
|
||
"light_p95": 6.0,
|
||
"light_p99": 9.0 if variant == "off" else 12.0,
|
||
"tempo_spans_per_s": {
|
||
"head": 10,
|
||
"tail": 20,
|
||
"balanced": 15,
|
||
}.get(variant, 100),
|
||
"prom_series": 1500
|
||
if variant in ("metrics", "balanced")
|
||
else 5000,
|
||
"prom_samples_per_s": 30
|
||
if variant in ("metrics", "balanced")
|
||
else 100,
|
||
"otel_collector_peak_mib": 90
|
||
if variant in ("tail", "balanced")
|
||
else 60,
|
||
"otel_collector_cpu": 0.1,
|
||
"tempo_cpu": 0.2
|
||
if variant in ("full", "full-aa", "metrics")
|
||
else 0.05,
|
||
"loki_cpu": 0.05,
|
||
"prometheus_cpu": 0.05,
|
||
"disk_tempo_bytes": 3e6,
|
||
"disk_loki_bytes": 1e6,
|
||
"disk_prometheus_bytes": 1e6,
|
||
"spans_gap": 0.001,
|
||
"q_read_p95_buckets": 0.02
|
||
if variant == "metrics"
|
||
else 0.0075,
|
||
"q_read_p95_log": 0.007,
|
||
**dict.fromkeys(LOSS_KEYS, 0),
|
||
"flags": "",
|
||
},
|
||
)
|
||
for cls in RARE:
|
||
client = 30
|
||
keep = 3 if variant == "head" else client
|
||
classes.append(
|
||
{
|
||
"variant": variant,
|
||
"class": cls,
|
||
"client": client,
|
||
"traces": keep,
|
||
"chains": keep,
|
||
"logs": client,
|
||
"logs_with_trace": client,
|
||
"log_to_trace": keep,
|
||
"metric": client,
|
||
"server_slow": client,
|
||
"server_slow_traces": keep,
|
||
"server_slow_linked": keep,
|
||
},
|
||
)
|
||
result = {v.name: v.result for v in verdicts(rows, classes, 256)}
|
||
assert result["B1"] == "подтверждено", result
|
||
assert result["B3"] == "подтверждено", result
|
||
assert result["H2"] == "подтверждено", [
|
||
v.render() for v in verdicts(rows, classes, 256) if v.name == "H2"
|
||
]
|
||
assert result["H4"] == "подтверждено", [
|
||
v.render() for v in verdicts(rows, classes, 256) if v.name == "H4"
|
||
]
|
||
assert result["H8"] == "подтверждено", result
|
||
text = render_summary(
|
||
{
|
||
"label": "self",
|
||
"profile": {"name": "telemetry", "repeats": 3},
|
||
"telemetry_cost": {"windows": [30, 90, 30]},
|
||
},
|
||
rows,
|
||
classes,
|
||
[],
|
||
)
|
||
assert "## Гипотезы и критерии" in text and "**B5:" in text
|
||
for name, svg in render_plots(rows, classes).items():
|
||
assert svg.startswith("<svg") and name.endswith(".svg")
|
||
print("telemetry_cost selftest: ok")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
if sys.argv[1:] == ["--selftest"]:
|
||
selftest()
|
||
sys.exit(0)
|
||
print(__doc__)
|
||
sys.exit(2)
|