456 lines
17 KiB
Python
456 lines
17 KiB
Python
"""Сравнение двух меток стенда: дельта по ячейкам матрицы.
|
||
|
||
Запуск — `make bench_compare BEFORE=<метка> AFTER=<метка> [AA=<метка>]`.
|
||
Читает `runs.csv` и `meta.json` из `evidence/bench/<метка>/`, пишет
|
||
`evidence/compare/<до>__<после>.md`. Только stdlib, docker и приложение
|
||
не нужны. Метки с разной методикой не сравниваются (с лабы 2 методика
|
||
включает и хранилища: design change `bench-postgres`, решение 8). Правило
|
||
«эффект или шум» — design change `perf-bottleneck-fix`, решение 7.
|
||
"""
|
||
|
||
import argparse
|
||
import csv
|
||
import json
|
||
import re
|
||
import statistics
|
||
import sys
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
BENCH_DIR = ROOT / "evidence" / "bench"
|
||
OUT_DIR = ROOT / "evidence" / "compare"
|
||
SCENARIOS = ("read", "write")
|
||
# ячейка матрицы: сценарий, размер истории, число клиентов
|
||
type CellKey = tuple[str, int, int]
|
||
|
||
|
||
class CompareError(Exception):
|
||
"""Метки нельзя сравнить: сообщение для человека."""
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Label:
|
||
name: str
|
||
meta: dict[str, Any]
|
||
runs: list[dict[str, str]]
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Stat:
|
||
"""Повторы одной ячейки: RPS, CPU на запрос, медианы задержек."""
|
||
|
||
rps: list[float]
|
||
cpu_us: float
|
||
p50: float
|
||
p95: float
|
||
p99: float
|
||
|
||
@property
|
||
def median(self) -> float:
|
||
return statistics.median(self.rps)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Row:
|
||
key: CellKey
|
||
before: Stat
|
||
after: Stat
|
||
|
||
@property
|
||
def delta(self) -> float:
|
||
"""Изменение медианы RPS, %."""
|
||
return (self.after.median / self.before.median - 1) * 100
|
||
|
||
@property
|
||
def overlap(self) -> bool:
|
||
"""Пересекаются ли разбросы min–max повторов двух меток."""
|
||
return min(self.before.rps) <= max(self.after.rps) and min(
|
||
self.after.rps,
|
||
) <= max(self.before.rps)
|
||
|
||
|
||
def human(n: int) -> str:
|
||
if n >= 1_000_000 and n % 1_000_000 == 0:
|
||
return f"{n // 1_000_000}M"
|
||
if n >= 1000 and n % 1000 == 0:
|
||
return f"{n // 1000}k"
|
||
return str(n)
|
||
|
||
|
||
def load(name: str) -> Label:
|
||
if not re.fullmatch(r"[A-Za-z0-9._-]+", name):
|
||
raise CompareError(f"bad label {name!r}: use [A-Za-z0-9._-]")
|
||
directory = BENCH_DIR / name
|
||
for file in ("meta.json", "runs.csv"):
|
||
if not (directory / file).is_file():
|
||
raise CompareError(
|
||
f"{(directory / file).relative_to(ROOT)} not found",
|
||
)
|
||
meta = json.loads((directory / "meta.json").read_text(encoding="utf-8"))
|
||
with (directory / "runs.csv").open(encoding="utf-8", newline="") as file:
|
||
runs = list(csv.DictReader(file))
|
||
return Label(name=name, meta=meta, runs=runs)
|
||
|
||
|
||
def flatten(prefix: str, value: Any, into: dict[str, Any]) -> None:
|
||
"""Вложенные словари паспорта → плоские ключи `a.b.c`."""
|
||
if isinstance(value, dict):
|
||
for key, inner in value.items():
|
||
flatten(f"{prefix}.{key}", inner, into)
|
||
else:
|
||
into[prefix] = value
|
||
|
||
|
||
def methodology(label: Label) -> dict[str, Any]:
|
||
"""Параметры, которые у сравнимых меток обязаны совпадать.
|
||
|
||
Профиль, лимиты всех контейнеров стенда, образ генератора, образы и
|
||
настройки PostgreSQL и Valkey, окружение приложения (решение 8 design
|
||
change `bench-postgres`). У Valkey настройки — `maxmemory*` из
|
||
`valkey.conf`: правку конфигурации образ не покажет.
|
||
"""
|
||
meta = label.meta
|
||
stand = meta.get("stand", {})
|
||
flat: dict[str, Any] = {}
|
||
flatten("profile", meta.get("profile", {}), flat)
|
||
flatten("limits", meta.get("limits", {}), flat)
|
||
for name, value in (
|
||
("generator.image", meta.get("generator", {}).get("image")),
|
||
("stand.postgres.image", stand.get("postgres", {}).get("image")),
|
||
("stand.valkey.image", stand.get("valkey", {}).get("image")),
|
||
):
|
||
if value is not None:
|
||
flat[name] = value
|
||
flatten(
|
||
"stand.postgres.settings",
|
||
stand.get("postgres", {}).get("settings", {}),
|
||
flat,
|
||
)
|
||
for key, value in stand.get("valkey", {}).items():
|
||
if key.startswith("maxmemory"):
|
||
flat[f"stand.valkey.{key}"] = value
|
||
flatten("stand.app_env", stand.get("app_env", {}), flat)
|
||
return flat
|
||
|
||
|
||
def check_methodology(first: Label, second: Label) -> list[str]:
|
||
"""Отказ при различии; возвращает параметры, записанные в одной метке."""
|
||
a, b = methodology(first), methodology(second)
|
||
differs = [
|
||
f"{key}: {first.name} = {a[key]!r}, {second.name} = {b[key]!r}"
|
||
for key in sorted(a.keys() & b.keys())
|
||
if a[key] != b[key]
|
||
]
|
||
if differs:
|
||
raise CompareError(
|
||
f"labels {first.name} and {second.name} were measured with "
|
||
"different methodology:\n " + "\n ".join(differs),
|
||
)
|
||
return [
|
||
f"`{key}` записан только в `{(first if key in a else second).name}`: "
|
||
"совпадение не проверено"
|
||
for key in sorted(a.keys() ^ b.keys())
|
||
]
|
||
|
||
|
||
def notes(label: Label) -> list[str]:
|
||
"""Что мешает считать метку чистым замером."""
|
||
meta = label.meta
|
||
found = []
|
||
if meta.get("status") != "complete":
|
||
found.append(f"статус {meta.get('status')}: {meta.get('reason')}")
|
||
if meta.get("git", {}).get("dirty"):
|
||
found.append("незакоммиченные изменения (`dirty: true`)")
|
||
if meta.get("overrides"):
|
||
found.append(f"отладочные переопределения {meta['overrides']}")
|
||
found += [f"хост: {text}" for text in meta.get("host_warnings", [])]
|
||
if meta.get("other_containers"):
|
||
found.append(
|
||
"посторонние контейнеры: " + ", ".join(meta["other_containers"]),
|
||
)
|
||
flagged = [
|
||
run["run"]
|
||
for run in label.runs
|
||
if run.get("flags")
|
||
or run.get("requests_failed", "0") not in ("0", "")
|
||
or run.get("check_failed", "0") not in ("0", "")
|
||
]
|
||
if flagged:
|
||
found.append("прогоны с пометками или ошибками: " + ", ".join(flagged))
|
||
return [f"`{label.name}`: {text}" for text in found]
|
||
|
||
|
||
def cells(label: Label) -> dict[CellKey, Stat]:
|
||
grouped: dict[CellKey, list[dict[str, str]]] = {}
|
||
for run in label.runs:
|
||
if not run["rps"] or float(run["rps"]) <= 0:
|
||
continue
|
||
key = (run["scenario"], int(run["size"]), int(run["concurrency"]))
|
||
grouped.setdefault(key, []).append(run)
|
||
|
||
def median(rows: list[dict[str, str]], column: str) -> float:
|
||
return statistics.median(float(row[column]) for row in rows)
|
||
|
||
return {
|
||
key: Stat(
|
||
rps=[float(row["rps"]) for row in rows],
|
||
cpu_us=statistics.median(
|
||
float(row["app_cpu_cores"]) / float(row["rps"]) * 1e6
|
||
for row in rows
|
||
),
|
||
p50=median(rows, "p50_ms"),
|
||
p95=median(rows, "p95_ms"),
|
||
p99=median(rows, "p99_ms"),
|
||
)
|
||
for key, rows in grouped.items()
|
||
}
|
||
|
||
|
||
def compare(before: Label, after: Label) -> tuple[list[Row], list[str]]:
|
||
"""Строки по общим ячейкам и замечания о ячейках одной метки."""
|
||
first, second = cells(before), cells(after)
|
||
rows = [
|
||
Row(key=key, before=first[key], after=second[key])
|
||
for key in first
|
||
if key in second
|
||
]
|
||
lonely = [
|
||
f"ячейка {key[0]} {human(key[1])} × {key[2]} есть только в "
|
||
f"`{(before if key in first else after).name}`"
|
||
for key in sorted(first.keys() ^ second.keys())
|
||
]
|
||
repeats = [
|
||
f"ячейка {row.key[0]} {human(row.key[1])} × {row.key[2]}: повторов "
|
||
f"{len(row.before.rps)} и {len(row.after.rps)}"
|
||
for row in rows
|
||
if len(row.before.rps) != len(row.after.rps)
|
||
]
|
||
return rows, lonely + repeats
|
||
|
||
|
||
def noise_floor(rows: list[Row]) -> dict[str, Row]:
|
||
"""Ячейка с наибольшим изменением медианы RPS в каждом сценарии."""
|
||
floor: dict[str, Row] = {}
|
||
for row in rows:
|
||
worst = floor.get(row.key[0])
|
||
if worst is None or abs(row.delta) > abs(worst.delta):
|
||
floor[row.key[0]] = row
|
||
return floor
|
||
|
||
|
||
def passport_rows(labels: list[Label]) -> list[str]:
|
||
def field(title: str, value: Any) -> str:
|
||
return (
|
||
f"| {title} | " + " | ".join(value(x.meta) for x in labels) + " |"
|
||
)
|
||
|
||
def host(meta: dict[str, Any]) -> str:
|
||
start = meta.get("host_state", {}).get("start", {})
|
||
return f"swap {start.get('swap_used_mib')} МиБ, {start.get('power')}"
|
||
|
||
def versions(meta: dict[str, Any]) -> str:
|
||
found = meta.get("app_image", {}).get("versions", {})
|
||
return ", ".join(f"{name} {ver}" for name, ver in found.items())
|
||
|
||
return [
|
||
"| | " + " | ".join(f"`{x.name}`" for x in labels) + " |",
|
||
"|---|" + "---|" * len(labels),
|
||
field("Commit", lambda m: f"`{m.get('git', {}).get('commit')}`"),
|
||
field("Начало прогона", lambda m: str(m.get("started_at"))),
|
||
field("Статус", lambda m: str(m.get("status"))),
|
||
field("Версии в образе", versions),
|
||
field("Хост в начале", host),
|
||
]
|
||
|
||
|
||
def scenario_table(
|
||
rows: list[Row],
|
||
floor: dict[str, Row] | None,
|
||
) -> list[str]:
|
||
lines = [
|
||
"| Размер | Клиенты | RPS до | RPS после | Δ RPS | Вывод | "
|
||
"CPU на запрос, мкс | Δ CPU, мкс | p50, мс | p95, мс | p99, мс |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for row in rows:
|
||
before, after = row.before, row.after
|
||
if floor is None:
|
||
verdict = (
|
||
"разбросы пересекаются"
|
||
if row.overlap
|
||
else "разбросы не пересекаются"
|
||
)
|
||
elif not row.overlap and abs(row.delta) > abs(floor[row.key[0]].delta):
|
||
verdict = "эффект"
|
||
else:
|
||
verdict = "в пределах шума"
|
||
lines.append(
|
||
f"| {human(row.key[1])} | {row.key[2]} | "
|
||
f"{before.median:.0f} ({min(before.rps):.0f}–"
|
||
f"{max(before.rps):.0f}) | "
|
||
f"{after.median:.0f} ({min(after.rps):.0f}–"
|
||
f"{max(after.rps):.0f}) | "
|
||
f"{row.delta:+.1f}% | {verdict} | "
|
||
f"{before.cpu_us:.0f} → {after.cpu_us:.0f} | "
|
||
f"{after.cpu_us - before.cpu_us:+.0f} | "
|
||
f"{before.p50:.2f} → {after.p50:.2f} | "
|
||
f"{before.p95:.2f} → {after.p95:.2f} | "
|
||
f"{before.p99:.2f} → {after.p99:.2f} |",
|
||
)
|
||
deltas = [row.delta for row in rows]
|
||
lines += [
|
||
"",
|
||
f"Δ RPS по ячейкам: от {min(deltas):+.1f}% до {max(deltas):+.1f}%, "
|
||
f"медиана {statistics.median(deltas):+.1f}%.",
|
||
]
|
||
return lines
|
||
|
||
|
||
def memory_table(before: Label, after: Label) -> list[str]:
|
||
def sizes(label: Label) -> dict[int, dict[str, Any]]:
|
||
return {item["size"]: item for item in label.meta.get("sizes", [])}
|
||
|
||
first, second = sizes(before), sizes(after)
|
||
lines = [
|
||
"| Размер | Байт RSS на сообщение до | после | "
|
||
"RSS после наполнения до, МиБ | после | "
|
||
"Байт в БД на сообщение до | после |",
|
||
"|---|---|---|---|---|---|---|",
|
||
]
|
||
for size in sorted(first.keys() & second.keys()):
|
||
a, b = first[size], second[size]
|
||
lines.append(
|
||
f"| {human(size)} | {a.get('bytes_per_message')} | "
|
||
f"{b.get('bytes_per_message')} | {a.get('rss_after_mib')} | "
|
||
f"{b.get('rss_after_mib')} | {a.get('db_bytes_per_message')} | "
|
||
f"{b.get('db_bytes_per_message')} |",
|
||
)
|
||
return lines
|
||
|
||
|
||
def render(
|
||
before: Label,
|
||
after: Label,
|
||
aa: Label | None,
|
||
command: str,
|
||
) -> str:
|
||
unverified = check_methodology(before, after)
|
||
rows, remarks = compare(before, after)
|
||
if not rows:
|
||
raise CompareError("labels have no common matrix cells")
|
||
labels = [before, after]
|
||
floor = None
|
||
if aa is not None:
|
||
unverified += check_methodology(aa, before)
|
||
aa_rows, aa_remarks = compare(aa, before)
|
||
floor = noise_floor(aa_rows)
|
||
missing = [s for s in {row.key[0] for row in rows} if s not in floor]
|
||
if missing:
|
||
raise CompareError(
|
||
f"label {aa.name} has no cells for {', '.join(missing)}",
|
||
)
|
||
remarks += aa_remarks
|
||
labels.append(aa)
|
||
remarks = (
|
||
unverified + [text for x in labels for text in notes(x)] + remarks
|
||
)
|
||
lines = [
|
||
f"# Сравнение меток: `{before.name}` → `{after.name}`",
|
||
"",
|
||
f"- Команда: `{command}`",
|
||
"- «До» и «после» — медиана повторов ячейки, в скобках min–max. "
|
||
"Δ RPS — отношение медиан минус единица.",
|
||
"- CPU на запрос — `app_cpu_cores / rps`, медиана по повторам; "
|
||
"задержки — медианы по повторам, «до → после».",
|
||
]
|
||
if aa is None:
|
||
lines.append(
|
||
"- Уровень шума не задан (`AA`): вывод говорит только, "
|
||
"пересекаются ли разбросы повторов.",
|
||
)
|
||
else:
|
||
lines += [
|
||
f"- Уровень шума — наибольшее по ячейкам сценария изменение "
|
||
f"медианы RPS между `{aa.name}` и `{before.name}`. Он имеет "
|
||
"смысл, только если код приложения в этих метках одинаков; "
|
||
"это проверяется по git, команда этого не видит.",
|
||
"- «Эффект» — разбросы повторов не пересекаются и изменение "
|
||
"медианы больше уровня шума сценария; иначе «в пределах шума».",
|
||
]
|
||
lines += [
|
||
"",
|
||
"## Метки",
|
||
"",
|
||
*passport_rows(labels),
|
||
"",
|
||
"## Замечания",
|
||
"",
|
||
]
|
||
lines += [f"- {text}" for text in dict.fromkeys(remarks)] or ["Нет."]
|
||
if floor is not None and aa is not None:
|
||
lines += [
|
||
"",
|
||
"## Уровень шума",
|
||
"",
|
||
f"| Сценарий | Наибольшее изменение RPS, `{aa.name}` → "
|
||
f"`{before.name}` | Ячейка |",
|
||
"|---|---|---|",
|
||
]
|
||
lines += [
|
||
f"| {scenario} | {abs(floor[scenario].delta):.1f}% | "
|
||
f"{human(floor[scenario].key[1])} × {floor[scenario].key[2]} |"
|
||
for scenario in SCENARIOS
|
||
if scenario in floor
|
||
]
|
||
for scenario in SCENARIOS:
|
||
selected = [row for row in rows if row.key[0] == scenario]
|
||
if selected:
|
||
lines += [
|
||
"",
|
||
f"## Сценарий `{scenario}`",
|
||
"",
|
||
*scenario_table(selected, floor),
|
||
]
|
||
lines += ["", "## Память", "", *memory_table(before, after)]
|
||
return "\n".join(lines) + "\n"
|
||
|
||
|
||
def main(argv: list[str]) -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
parser.add_argument("--before", default="")
|
||
parser.add_argument("--after", default="")
|
||
parser.add_argument("--aa", default="")
|
||
args = parser.parse_args(argv)
|
||
if not args.before or not args.after:
|
||
print(
|
||
"both labels are required: "
|
||
"make bench_compare BEFORE=<label> AFTER=<label> [AA=<label>]",
|
||
flush=True,
|
||
)
|
||
return 2
|
||
command = f"make bench_compare BEFORE={args.before} AFTER={args.after}"
|
||
if args.aa:
|
||
command += f" AA={args.aa}"
|
||
try:
|
||
text = render(
|
||
load(args.before),
|
||
load(args.after),
|
||
load(args.aa) if args.aa else None,
|
||
command,
|
||
)
|
||
except CompareError as exc:
|
||
print(f"compare failed: {exc}", flush=True)
|
||
return 1
|
||
out = OUT_DIR / f"{args.before}__{args.after}.md"
|
||
out.parent.mkdir(parents=True, exist_ok=True)
|
||
out.write_text(text, encoding="utf-8")
|
||
print(text, flush=True)
|
||
print(f"saved to {out.relative_to(ROOT)}", flush=True)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main(sys.argv[1:]))
|