hang-key names its trigger per run, turns SIGTERM and SIGHUP into an interrupt so the trigger is always dropped and the replica unpaused, and prints the cleanup SQL if dropping fails. idle-conns reports who closed the idle connections and can read the pool of another LB container, so a variant LB config can be checked against the same replicas. bench_compare refuses a label measured without the load balancer. Stale numbers removed from the haproxy.cfg comment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
470 lines
18 KiB
Python
470 lines
18 KiB
Python
"""Сравнение двух меток стенда: дельта по ячейкам матрицы.
|
||
|
||
Запуск — `make bench_compare BEFORE=<метка> AFTER=<метка> [AA=<метка>]`.
|
||
Читает `runs.csv` и `meta.json` из `evidence/bench/<метка>/`, пишет
|
||
`evidence/compare/<до>__<после>.md`. Только stdlib, docker и приложение
|
||
не нужны. Метки с разной методикой не сравниваются (с лабы 2 методика
|
||
включает и хранилища: design change `bench-postgres`, решение 8). Правило
|
||
«эффект или шум» — design change `perf-bottleneck-fix`, решение 7.
|
||
"""
|
||
|
||
import argparse
|
||
import csv
|
||
import json
|
||
import re
|
||
import statistics
|
||
import sys
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
BENCH_DIR = ROOT / "evidence" / "bench"
|
||
OUT_DIR = ROOT / "evidence" / "compare"
|
||
SCENARIOS = ("read", "write")
|
||
# ячейка матрицы: сценарий, размер истории, число клиентов
|
||
type CellKey = tuple[str, int, int]
|
||
|
||
|
||
class CompareError(Exception):
|
||
"""Метки нельзя сравнить: сообщение для человека."""
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Label:
|
||
name: str
|
||
meta: dict[str, Any]
|
||
runs: list[dict[str, str]]
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Stat:
|
||
"""Повторы одной ячейки: RPS, CPU на запрос, медианы задержек."""
|
||
|
||
rps: list[float]
|
||
cpu_us: float
|
||
p50: float
|
||
p95: float
|
||
p99: float
|
||
|
||
@property
|
||
def median(self) -> float:
|
||
return statistics.median(self.rps)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Row:
|
||
key: CellKey
|
||
before: Stat
|
||
after: Stat
|
||
|
||
@property
|
||
def delta(self) -> float:
|
||
"""Изменение медианы RPS, %."""
|
||
return (self.after.median / self.before.median - 1) * 100
|
||
|
||
@property
|
||
def overlap(self) -> bool:
|
||
"""Пересекаются ли разбросы min–max повторов двух меток."""
|
||
return min(self.before.rps) <= max(self.after.rps) and min(
|
||
self.after.rps,
|
||
) <= max(self.before.rps)
|
||
|
||
|
||
def human(n: int) -> str:
|
||
if n >= 1_000_000 and n % 1_000_000 == 0:
|
||
return f"{n // 1_000_000}M"
|
||
if n >= 1000 and n % 1000 == 0:
|
||
return f"{n // 1000}k"
|
||
return str(n)
|
||
|
||
|
||
def load(name: str) -> Label:
|
||
if not re.fullmatch(r"[A-Za-z0-9._-]+", name):
|
||
raise CompareError(f"bad label {name!r}: use [A-Za-z0-9._-]")
|
||
directory = BENCH_DIR / name
|
||
for file in ("meta.json", "runs.csv"):
|
||
if not (directory / file).is_file():
|
||
raise CompareError(
|
||
f"{(directory / file).relative_to(ROOT)} not found",
|
||
)
|
||
meta = json.loads((directory / "meta.json").read_text(encoding="utf-8"))
|
||
with (directory / "runs.csv").open(encoding="utf-8", newline="") as file:
|
||
runs = list(csv.DictReader(file))
|
||
return Label(name=name, meta=meta, runs=runs)
|
||
|
||
|
||
def flatten(prefix: str, value: Any, into: dict[str, Any]) -> None:
|
||
"""Вложенные словари паспорта → плоские ключи `a.b.c`."""
|
||
if isinstance(value, dict):
|
||
for key, inner in value.items():
|
||
flatten(f"{prefix}.{key}", inner, into)
|
||
else:
|
||
into[prefix] = value
|
||
|
||
|
||
def methodology(label: Label) -> dict[str, Any]:
|
||
"""Параметры, которые у сравнимых меток обязаны совпадать.
|
||
|
||
Профиль, лимиты всех контейнеров стенда, образ генератора, образы и
|
||
настройки PostgreSQL и Valkey, окружение приложения (решение 8 design
|
||
change `bench-postgres`). У Valkey настройки — `maxmemory*` из
|
||
`valkey.conf`: правку конфигурации образ не покажет.
|
||
"""
|
||
meta = label.meta
|
||
stand = meta.get("stand", {})
|
||
flat: dict[str, Any] = {}
|
||
flatten("profile", meta.get("profile", {}), flat)
|
||
flatten("limits", meta.get("limits", {}), flat)
|
||
for name, value in (
|
||
("generator.image", meta.get("generator", {}).get("image")),
|
||
("stand.postgres.image", stand.get("postgres", {}).get("image")),
|
||
("stand.valkey.image", stand.get("valkey", {}).get("image")),
|
||
):
|
||
if value is not None:
|
||
flat[name] = value
|
||
flatten(
|
||
"stand.postgres.settings",
|
||
stand.get("postgres", {}).get("settings", {}),
|
||
flat,
|
||
)
|
||
for key, value in stand.get("valkey", {}).items():
|
||
if key.startswith("maxmemory"):
|
||
flat[f"stand.valkey.{key}"] = value
|
||
flatten("stand.app_env", stand.get("app_env", {}), flat)
|
||
return flat
|
||
|
||
|
||
def check_methodology(first: Label, second: Label) -> list[str]:
|
||
"""Отказ при различии; возвращает параметры, записанные в одной метке."""
|
||
a, b = methodology(first), methodology(second)
|
||
# с change graceful-failover в пути запроса стоит HAProxy, и паспорт
|
||
# пишет число реплик; метка без него снята мимо LB — другая методика
|
||
# (решение 6 design change graceful-failover), а не «параметр
|
||
# записан только в одной метке»
|
||
without_lb = [
|
||
label.name
|
||
for label, flat in ((first, a), (second, b))
|
||
if "limits.api_replicas" not in flat
|
||
]
|
||
if len(without_lb) == 1:
|
||
raise CompareError(
|
||
f"label {without_lb[0]} was measured without the load balancer "
|
||
"(before change graceful-failover): different methodology",
|
||
)
|
||
differs = [
|
||
f"{key}: {first.name} = {a[key]!r}, {second.name} = {b[key]!r}"
|
||
for key in sorted(a.keys() & b.keys())
|
||
if a[key] != b[key]
|
||
]
|
||
if differs:
|
||
raise CompareError(
|
||
f"labels {first.name} and {second.name} were measured with "
|
||
"different methodology:\n " + "\n ".join(differs),
|
||
)
|
||
return [
|
||
f"`{key}` записан только в `{(first if key in a else second).name}`: "
|
||
"совпадение не проверено"
|
||
for key in sorted(a.keys() ^ b.keys())
|
||
]
|
||
|
||
|
||
def notes(label: Label) -> list[str]:
|
||
"""Что мешает считать метку чистым замером."""
|
||
meta = label.meta
|
||
found = []
|
||
if meta.get("status") != "complete":
|
||
found.append(f"статус {meta.get('status')}: {meta.get('reason')}")
|
||
if meta.get("git", {}).get("dirty"):
|
||
found.append("незакоммиченные изменения (`dirty: true`)")
|
||
if meta.get("overrides"):
|
||
found.append(f"отладочные переопределения {meta['overrides']}")
|
||
found += [f"хост: {text}" for text in meta.get("host_warnings", [])]
|
||
if meta.get("other_containers"):
|
||
found.append(
|
||
"посторонние контейнеры: " + ", ".join(meta["other_containers"]),
|
||
)
|
||
flagged = [
|
||
run["run"]
|
||
for run in label.runs
|
||
if run.get("flags")
|
||
or run.get("requests_failed", "0") not in ("0", "")
|
||
or run.get("check_failed", "0") not in ("0", "")
|
||
]
|
||
if flagged:
|
||
found.append("прогоны с пометками или ошибками: " + ", ".join(flagged))
|
||
return [f"`{label.name}`: {text}" for text in found]
|
||
|
||
|
||
def cells(label: Label) -> dict[CellKey, Stat]:
|
||
grouped: dict[CellKey, list[dict[str, str]]] = {}
|
||
for run in label.runs:
|
||
if not run["rps"] or float(run["rps"]) <= 0:
|
||
continue
|
||
key = (run["scenario"], int(run["size"]), int(run["concurrency"]))
|
||
grouped.setdefault(key, []).append(run)
|
||
|
||
def median(rows: list[dict[str, str]], column: str) -> float:
|
||
return statistics.median(float(row[column]) for row in rows)
|
||
|
||
return {
|
||
key: Stat(
|
||
rps=[float(row["rps"]) for row in rows],
|
||
cpu_us=statistics.median(
|
||
float(row["app_cpu_cores"]) / float(row["rps"]) * 1e6
|
||
for row in rows
|
||
),
|
||
p50=median(rows, "p50_ms"),
|
||
p95=median(rows, "p95_ms"),
|
||
p99=median(rows, "p99_ms"),
|
||
)
|
||
for key, rows in grouped.items()
|
||
}
|
||
|
||
|
||
def compare(before: Label, after: Label) -> tuple[list[Row], list[str]]:
|
||
"""Строки по общим ячейкам и замечания о ячейках одной метки."""
|
||
first, second = cells(before), cells(after)
|
||
rows = [
|
||
Row(key=key, before=first[key], after=second[key])
|
||
for key in first
|
||
if key in second
|
||
]
|
||
lonely = [
|
||
f"ячейка {key[0]} {human(key[1])} × {key[2]} есть только в "
|
||
f"`{(before if key in first else after).name}`"
|
||
for key in sorted(first.keys() ^ second.keys())
|
||
]
|
||
repeats = [
|
||
f"ячейка {row.key[0]} {human(row.key[1])} × {row.key[2]}: повторов "
|
||
f"{len(row.before.rps)} и {len(row.after.rps)}"
|
||
for row in rows
|
||
if len(row.before.rps) != len(row.after.rps)
|
||
]
|
||
return rows, lonely + repeats
|
||
|
||
|
||
def noise_floor(rows: list[Row]) -> dict[str, Row]:
|
||
"""Ячейка с наибольшим изменением медианы RPS в каждом сценарии."""
|
||
floor: dict[str, Row] = {}
|
||
for row in rows:
|
||
worst = floor.get(row.key[0])
|
||
if worst is None or abs(row.delta) > abs(worst.delta):
|
||
floor[row.key[0]] = row
|
||
return floor
|
||
|
||
|
||
def passport_rows(labels: list[Label]) -> list[str]:
|
||
def field(title: str, value: Any) -> str:
|
||
return (
|
||
f"| {title} | " + " | ".join(value(x.meta) for x in labels) + " |"
|
||
)
|
||
|
||
def host(meta: dict[str, Any]) -> str:
|
||
start = meta.get("host_state", {}).get("start", {})
|
||
return f"swap {start.get('swap_used_mib')} МиБ, {start.get('power')}"
|
||
|
||
def versions(meta: dict[str, Any]) -> str:
|
||
found = meta.get("app_image", {}).get("versions", {})
|
||
return ", ".join(f"{name} {ver}" for name, ver in found.items())
|
||
|
||
return [
|
||
"| | " + " | ".join(f"`{x.name}`" for x in labels) + " |",
|
||
"|---|" + "---|" * len(labels),
|
||
field("Commit", lambda m: f"`{m.get('git', {}).get('commit')}`"),
|
||
field("Начало прогона", lambda m: str(m.get("started_at"))),
|
||
field("Статус", lambda m: str(m.get("status"))),
|
||
field("Версии в образе", versions),
|
||
field("Хост в начале", host),
|
||
]
|
||
|
||
|
||
def scenario_table(
|
||
rows: list[Row],
|
||
floor: dict[str, Row] | None,
|
||
) -> list[str]:
|
||
lines = [
|
||
"| Размер | Клиенты | RPS до | RPS после | Δ RPS | Вывод | "
|
||
"CPU на запрос, мкс | Δ CPU, мкс | p50, мс | p95, мс | p99, мс |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for row in rows:
|
||
before, after = row.before, row.after
|
||
if floor is None:
|
||
verdict = (
|
||
"разбросы пересекаются"
|
||
if row.overlap
|
||
else "разбросы не пересекаются"
|
||
)
|
||
elif not row.overlap and abs(row.delta) > abs(floor[row.key[0]].delta):
|
||
verdict = "эффект"
|
||
else:
|
||
verdict = "в пределах шума"
|
||
lines.append(
|
||
f"| {human(row.key[1])} | {row.key[2]} | "
|
||
f"{before.median:.0f} ({min(before.rps):.0f}–"
|
||
f"{max(before.rps):.0f}) | "
|
||
f"{after.median:.0f} ({min(after.rps):.0f}–"
|
||
f"{max(after.rps):.0f}) | "
|
||
f"{row.delta:+.1f}% | {verdict} | "
|
||
f"{before.cpu_us:.0f} → {after.cpu_us:.0f} | "
|
||
f"{after.cpu_us - before.cpu_us:+.0f} | "
|
||
f"{before.p50:.2f} → {after.p50:.2f} | "
|
||
f"{before.p95:.2f} → {after.p95:.2f} | "
|
||
f"{before.p99:.2f} → {after.p99:.2f} |",
|
||
)
|
||
deltas = [row.delta for row in rows]
|
||
lines += [
|
||
"",
|
||
f"Δ RPS по ячейкам: от {min(deltas):+.1f}% до {max(deltas):+.1f}%, "
|
||
f"медиана {statistics.median(deltas):+.1f}%.",
|
||
]
|
||
return lines
|
||
|
||
|
||
def memory_table(before: Label, after: Label) -> list[str]:
|
||
def sizes(label: Label) -> dict[int, dict[str, Any]]:
|
||
return {item["size"]: item for item in label.meta.get("sizes", [])}
|
||
|
||
first, second = sizes(before), sizes(after)
|
||
lines = [
|
||
"| Размер | Байт RSS на сообщение до | после | "
|
||
"RSS после наполнения до, МиБ | после | "
|
||
"Байт в БД на сообщение до | после |",
|
||
"|---|---|---|---|---|---|---|",
|
||
]
|
||
for size in sorted(first.keys() & second.keys()):
|
||
a, b = first[size], second[size]
|
||
lines.append(
|
||
f"| {human(size)} | {a.get('bytes_per_message')} | "
|
||
f"{b.get('bytes_per_message')} | {a.get('rss_after_mib')} | "
|
||
f"{b.get('rss_after_mib')} | {a.get('db_bytes_per_message')} | "
|
||
f"{b.get('db_bytes_per_message')} |",
|
||
)
|
||
return lines
|
||
|
||
|
||
def render(
|
||
before: Label,
|
||
after: Label,
|
||
aa: Label | None,
|
||
command: str,
|
||
) -> str:
|
||
unverified = check_methodology(before, after)
|
||
rows, remarks = compare(before, after)
|
||
if not rows:
|
||
raise CompareError("labels have no common matrix cells")
|
||
labels = [before, after]
|
||
floor = None
|
||
if aa is not None:
|
||
unverified += check_methodology(aa, before)
|
||
aa_rows, aa_remarks = compare(aa, before)
|
||
floor = noise_floor(aa_rows)
|
||
missing = [s for s in {row.key[0] for row in rows} if s not in floor]
|
||
if missing:
|
||
raise CompareError(
|
||
f"label {aa.name} has no cells for {', '.join(missing)}",
|
||
)
|
||
remarks += aa_remarks
|
||
labels.append(aa)
|
||
remarks = (
|
||
unverified + [text for x in labels for text in notes(x)] + remarks
|
||
)
|
||
lines = [
|
||
f"# Сравнение меток: `{before.name}` → `{after.name}`",
|
||
"",
|
||
f"- Команда: `{command}`",
|
||
"- «До» и «после» — медиана повторов ячейки, в скобках min–max. "
|
||
"Δ RPS — отношение медиан минус единица.",
|
||
"- CPU на запрос — `app_cpu_cores / rps`, медиана по повторам; "
|
||
"задержки — медианы по повторам, «до → после».",
|
||
]
|
||
if aa is None:
|
||
lines.append(
|
||
"- Уровень шума не задан (`AA`): вывод говорит только, "
|
||
"пересекаются ли разбросы повторов.",
|
||
)
|
||
else:
|
||
lines += [
|
||
f"- Уровень шума — наибольшее по ячейкам сценария изменение "
|
||
f"медианы RPS между `{aa.name}` и `{before.name}`. Он имеет "
|
||
"смысл, только если код приложения в этих метках одинаков; "
|
||
"это проверяется по git, команда этого не видит.",
|
||
"- «Эффект» — разбросы повторов не пересекаются и изменение "
|
||
"медианы больше уровня шума сценария; иначе «в пределах шума».",
|
||
]
|
||
lines += [
|
||
"",
|
||
"## Метки",
|
||
"",
|
||
*passport_rows(labels),
|
||
"",
|
||
"## Замечания",
|
||
"",
|
||
]
|
||
lines += [f"- {text}" for text in dict.fromkeys(remarks)] or ["Нет."]
|
||
if floor is not None and aa is not None:
|
||
lines += [
|
||
"",
|
||
"## Уровень шума",
|
||
"",
|
||
f"| Сценарий | Наибольшее изменение RPS, `{aa.name}` → "
|
||
f"`{before.name}` | Ячейка |",
|
||
"|---|---|---|",
|
||
]
|
||
lines += [
|
||
f"| {scenario} | {abs(floor[scenario].delta):.1f}% | "
|
||
f"{human(floor[scenario].key[1])} × {floor[scenario].key[2]} |"
|
||
for scenario in SCENARIOS
|
||
if scenario in floor
|
||
]
|
||
for scenario in SCENARIOS:
|
||
selected = [row for row in rows if row.key[0] == scenario]
|
||
if selected:
|
||
lines += [
|
||
"",
|
||
f"## Сценарий `{scenario}`",
|
||
"",
|
||
*scenario_table(selected, floor),
|
||
]
|
||
lines += ["", "## Память", "", *memory_table(before, after)]
|
||
return "\n".join(lines) + "\n"
|
||
|
||
|
||
def main(argv: list[str]) -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
parser.add_argument("--before", default="")
|
||
parser.add_argument("--after", default="")
|
||
parser.add_argument("--aa", default="")
|
||
args = parser.parse_args(argv)
|
||
if not args.before or not args.after:
|
||
print(
|
||
"both labels are required: "
|
||
"make bench_compare BEFORE=<label> AFTER=<label> [AA=<label>]",
|
||
flush=True,
|
||
)
|
||
return 2
|
||
command = f"make bench_compare BEFORE={args.before} AFTER={args.after}"
|
||
if args.aa:
|
||
command += f" AA={args.aa}"
|
||
try:
|
||
text = render(
|
||
load(args.before),
|
||
load(args.after),
|
||
load(args.aa) if args.aa else None,
|
||
command,
|
||
)
|
||
except CompareError as exc:
|
||
print(f"compare failed: {exc}", flush=True)
|
||
return 1
|
||
out = OUT_DIR / f"{args.before}__{args.after}.md"
|
||
out.parent.mkdir(parents=True, exist_ok=True)
|
||
out.write_text(text, encoding="utf-8")
|
||
print(text, flush=True)
|
||
print(f"saved to {out.relative_to(ROOT)}", flush=True)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main(sys.argv[1:]))
|