249 lines
10 KiB
Python
249 lines
10 KiB
Python
"""Сравнение двух меток серий трека 1: «до» и «после» одного изменения.
|
||
|
||
uv run python bench/compare.py --before scaling-ab-before \\
|
||
--after scaling-ab-after [--aa scaling-ab-aa]
|
||
→ evidence/compare/<до>__<после>.md
|
||
|
||
По общим конфигурациям: R* (медиана, min–max, Δ), задержки на
|
||
фиксированных уровнях, CPU на запрос реплик и PostgreSQL, новые сессии
|
||
PostgreSQL и соединения на ступени R* и на первой нарушившей. С `--aa`
|
||
(метка того же кода, что «до») — уровень шума: наибольшее |Δ медианы R*|
|
||
между «до» и A/A по общим конфигурациям.
|
||
|
||
Изменение R* — эффект, только если: разбросы повторов не пересекаются,
|
||
|Δ медианы| больше шума A/A и не меньше двух ступеней сетки поиска
|
||
(R* после / R* до ≥ √factor ≈ 1,118 при factor 1,25: шаг сетки после двух
|
||
делений — factor^¼, одна ступень — в пределах разрешения поиска). Иначе
|
||
— «шум» или «в пределах разрешения».
|
||
|
||
Методика меток должна совпадать: профиль (кроме списка ячеек серии ab),
|
||
лимиты, образы хранилищ, хост. Различаться может только код приложения
|
||
(`src_sha256`). Иначе — отказ.
|
||
"""
|
||
|
||
import argparse
|
||
import csv
|
||
import json
|
||
import math
|
||
import statistics
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
SCALING = ROOT / "evidence" / "scaling"
|
||
OUT = ROOT / "evidence" / "compare"
|
||
|
||
|
||
def read_csv(path: Path) -> list[dict]:
|
||
with path.open(encoding="utf-8") as handle:
|
||
return list(csv.DictReader(handle))
|
||
|
||
|
||
def num(value) -> float | None:
|
||
try:
|
||
return float(value)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def load(label: str) -> dict:
|
||
folder = SCALING / label
|
||
if not (folder / "meta.json").exists():
|
||
raise SystemExit(f"no label {label} in evidence/scaling")
|
||
return {
|
||
"label": label,
|
||
"meta": json.loads((folder / "meta.json").read_text(encoding="utf-8")),
|
||
"searches": read_csv(folder / "searches.csv"),
|
||
"runs": read_csv(folder / "runs.csv"),
|
||
}
|
||
|
||
|
||
def methodology(meta: dict) -> dict:
|
||
"""Что должно совпасть у сравниваемых меток."""
|
||
profile = dict(meta["profile"])
|
||
profile.pop("configs", None)
|
||
stand = meta.get("stand") or {}
|
||
return {
|
||
"profile": profile,
|
||
"series_name": profile.get("name"),
|
||
"configs": meta.get("configs"),
|
||
"cpu_model": meta["host"].get("cpu_model"),
|
||
"vm_cpus": meta["docker"].get("NCPU"),
|
||
"postgres_image": (stand.get("postgres") or {}).get("image"),
|
||
}
|
||
|
||
|
||
def check_methodology(before: dict, after: dict) -> list[str]:
|
||
a, b = methodology(before["meta"]), methodology(after["meta"])
|
||
problems = []
|
||
if a["profile"] != b["profile"]:
|
||
problems.append("profile differs")
|
||
if a["cpu_model"] != b["cpu_model"] or a["vm_cpus"] != b["vm_cpus"]:
|
||
problems.append("host or docker VM differs")
|
||
common = set(a["configs"] or {}) & set(b["configs"] or {})
|
||
for config in sorted(common):
|
||
if a["configs"][config] != b["configs"][config]:
|
||
problems.append(f"limits of {config} differ")
|
||
return problems
|
||
|
||
|
||
def stats_by_config(data: dict) -> dict:
|
||
out = {}
|
||
for row in data["searches"]:
|
||
entry = out.setdefault(row["config"], {"r": [], "cells": []})
|
||
if num(row["r_star"]) is not None:
|
||
entry["r"].append(num(row["r_star"]))
|
||
entry["cells"].append(row)
|
||
for entry in out.values():
|
||
values = entry["r"] or [0.0]
|
||
entry["median"] = statistics.median(values)
|
||
entry["min"], entry["max"] = min(values), max(values)
|
||
return out
|
||
|
||
|
||
def step_metric(data: dict, config: str, which: str, key: str) -> float | None:
|
||
"""Медиана поля на ступени R* (`star`) или первой нарушившей (`fail`)."""
|
||
values = []
|
||
for search in data["searches"]:
|
||
if search["config"] != config:
|
||
continue
|
||
rate = num(search["r_star"] if which == "star" else search["first_fail"])
|
||
row = next(
|
||
(
|
||
r
|
||
for r in data["runs"]
|
||
if r["cell"] == search["cell"]
|
||
and r["phase"] != "fixed"
|
||
and num(r["rate"]) == rate
|
||
),
|
||
None,
|
||
)
|
||
if row is not None and num(row.get(key)) is not None:
|
||
values.append(num(row[key]))
|
||
return statistics.median(values) if values else None
|
||
|
||
|
||
def fixed_metric(data: dict, config: str, rate: float, key: str) -> float | None:
|
||
values = [
|
||
num(r[key])
|
||
for r in data["runs"]
|
||
if r["config"] == config
|
||
and r["phase"] == "fixed"
|
||
and num(r["rate"]) == rate
|
||
and num(r.get(key)) is not None
|
||
]
|
||
return statistics.median(values) if values else None
|
||
|
||
|
||
def fmt(value: float | None, digits: int = 1) -> str:
|
||
if value is None:
|
||
return "—"
|
||
return f"{round(value, digits):g}"
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
parser.add_argument("--before", required=True)
|
||
parser.add_argument("--after", required=True)
|
||
parser.add_argument("--aa")
|
||
args = parser.parse_args()
|
||
before, after = load(args.before), load(args.after)
|
||
aa = load(args.aa) if args.aa else None
|
||
problems = check_methodology(before, after)
|
||
if aa:
|
||
problems += [f"A/A: {p}" for p in check_methodology(before, aa)]
|
||
if aa["meta"]["code"]["src_sha256"] != before["meta"]["code"]["src_sha256"]:
|
||
problems.append("A/A label has different code than BEFORE (src_sha256)")
|
||
if problems:
|
||
raise SystemExit("labels are not comparable: " + "; ".join(problems))
|
||
|
||
factor = before["meta"]["profile"]["ladder_factor"]
|
||
two_steps = math.sqrt(factor)
|
||
sb, sa = stats_by_config(before), stats_by_config(after)
|
||
common = [c for c in before["meta"]["profile"]["configs"] if c in sb and c in sa]
|
||
noise = None
|
||
if aa:
|
||
saa = stats_by_config(aa)
|
||
deltas = [
|
||
abs(saa[c]["median"] - sb[c]["median"]) / sb[c]["median"]
|
||
for c in common
|
||
if c in saa and sb[c]["median"]
|
||
]
|
||
noise = max(deltas) if deltas else None
|
||
|
||
same_code = (
|
||
before["meta"]["code"]["src_sha256"] == after["meta"]["code"]["src_sha256"]
|
||
)
|
||
lines = [
|
||
f"# Сравнение: {args.before} → {args.after}",
|
||
"",
|
||
f"- Код «до»: src_sha256 `{before['meta']['code']['src_sha256'][:16]}`; «после»: "
|
||
f"`{after['meta']['code']['src_sha256'][:16]}`"
|
||
+ (" — **код одинаковый: это A/A, а не изменение**" if same_code else "")
|
||
+ ".",
|
||
f"- Пул приложения «до»: {before['meta']['image'].get('app_defaults')}; «после»: "
|
||
f"{after['meta']['image'].get('app_defaults')}.",
|
||
f"- Шум A/A ({args.aa}): "
|
||
+ (
|
||
f"наибольшее |Δ медианы R*| = {noise * 100:.1f} %"
|
||
if noise is not None
|
||
else "не задан"
|
||
)
|
||
+ ".",
|
||
f"- Порог эффекта: разбросы не пересекаются, |Δ| > шума и R* после / до ≥ "
|
||
f"{two_steps:.3f} (две ступени сетки поиска; одна ступень — {factor**0.25:.3f}).",
|
||
"",
|
||
"| Конфигурация | R* до, медиана (min–max) | R* после | Δ | Вывод | p95 до / после на фиксированных, мс | API мкс/запрос до / после | PostgreSQL мкс/запрос на R* до / после | Новых сессий за окно на первой нарушившей до / после | Соединений на R* до / после |",
|
||
"|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
rates = before["meta"]["profile"]["fixed_rates"]
|
||
for config in common:
|
||
b, a = sb[config], sa[config]
|
||
delta = (a["median"] - b["median"]) / b["median"] if b["median"] else None
|
||
overlap = not (a["min"] > b["max"] or a["max"] < b["min"])
|
||
ratio = a["median"] / b["median"] if b["median"] else None
|
||
if delta is None:
|
||
verdict = "нет данных"
|
||
elif overlap:
|
||
verdict = "шум (разбросы пересекаются)"
|
||
elif noise is not None and abs(delta) <= noise:
|
||
verdict = "шум (в пределах A/A)"
|
||
elif ratio is not None and (ratio >= two_steps or ratio <= 1 / two_steps):
|
||
verdict = "эффект"
|
||
else:
|
||
verdict = "в пределах разрешения поиска"
|
||
p95 = " ; ".join(
|
||
f"{r:g}: {fmt(fixed_metric(before, config, r, 'p95_ms'))} / {fmt(fixed_metric(after, config, r, 'p95_ms'))}"
|
||
for r in rates
|
||
)
|
||
api_us = (
|
||
f"{fmt(fixed_metric(before, config, rates[-1], 'api_cpu_us_per_req'))} / "
|
||
f"{fmt(fixed_metric(after, config, rates[-1], 'api_cpu_us_per_req'))}"
|
||
)
|
||
db_us = (
|
||
f"{fmt(step_metric(before, config, 'star', 'db_cpu_us_per_req'))} / "
|
||
f"{fmt(step_metric(after, config, 'star', 'db_cpu_us_per_req'))}"
|
||
)
|
||
sessions = (
|
||
f"{fmt(step_metric(before, config, 'fail', 'pg_new_sessions'), 0)} / "
|
||
f"{fmt(step_metric(after, config, 'fail', 'pg_new_sessions'), 0)}"
|
||
)
|
||
conns = (
|
||
f"{fmt(step_metric(before, config, 'star', 'pg_conn_max'), 0)} / "
|
||
f"{fmt(step_metric(after, config, 'star', 'pg_conn_max'), 0)}"
|
||
)
|
||
lines.append(
|
||
f"| {config} | {b['median']:g} ({b['min']:g}–{b['max']:g}) | "
|
||
f"{a['median']:g} ({a['min']:g}–{a['max']:g}) | "
|
||
f"{fmt(delta * 100 if delta is not None else None)} % | {verdict} | {p95} | "
|
||
f"{api_us} | {db_us} | {sessions} | {conns} |"
|
||
)
|
||
OUT.mkdir(parents=True, exist_ok=True)
|
||
path = OUT / f"{args.before}__{args.after}.md"
|
||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
print("\n".join(lines))
|
||
print(f"\nwritten: {path.relative_to(ROOT)}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|