384 lines
15 KiB
Python
384 lines
15 KiB
Python
import argparse
|
||
import collections
|
||
import csv
|
||
import statistics
|
||
import subprocess
|
||
import time
|
||
from pathlib import Path
|
||
|
||
import matplotlib.pyplot as plt
|
||
|
||
ORDER = ("1x1", "2x1", "4x1", "8x1", "2x2", "1x4")
|
||
PALETTE = ("#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300", "#6250d6", "#e34948")
|
||
MARKERS = ("o", "s", "^", "D", "v", "P", "X", "*")
|
||
USL_POINTS = 3
|
||
MIN_REPEATS = 3
|
||
COMPONENTS = (("api", "API"), ("postgres", "PostgreSQL"), ("lb", "HAProxy"), ("k6", "k6"))
|
||
INK, MUTED, GRID, SURFACE = "#0b0b0b", "#52514e", "#e4e3df", "#fcfcfb"
|
||
STAMP = (
|
||
f"date: {time.strftime('%FT%TZ', time.gmtime())} commit: "
|
||
+ subprocess.check_output(["/usr/bin/env", "git", "rev-parse", "HEAD"], text=True).strip()
|
||
)
|
||
SUMMARY = (
|
||
"series variant config rates capacity_rate capacity_label "
|
||
"capacity_ok_rps capacity_p95_ms capacity_p99_ms "
|
||
"peak_rate peak_ok_rps peak_ok_min peak_ok_max peak_p95_ms peak_failed_ratio peak_dropped "
|
||
"cpu_api cpu_api_limit cpu_postgres cpu_lb cpu_k6 pg_active_avg pg_lock_avg pg_idle_tx_avg "
|
||
"pg_idle_avg pool_total pg_backends_max lb_qcur_max"
|
||
).split()
|
||
|
||
plt.rcParams.update(
|
||
{
|
||
"figure.facecolor": SURFACE,
|
||
"axes.facecolor": SURFACE,
|
||
"axes.edgecolor": MUTED,
|
||
"axes.labelcolor": MUTED,
|
||
"axes.titlecolor": INK,
|
||
"axes.titleweight": "bold",
|
||
"axes.grid": True,
|
||
"grid.color": GRID,
|
||
"grid.linewidth": 0.8,
|
||
"axes.spines.top": False,
|
||
"axes.spines.right": False,
|
||
"xtick.color": MUTED,
|
||
"ytick.color": MUTED,
|
||
"legend.frameon": False,
|
||
"lines.linewidth": 2,
|
||
"lines.markersize": 7,
|
||
"font.size": 10,
|
||
}
|
||
)
|
||
|
||
|
||
VARIANTS = {
|
||
"before": {"color": MUTED, "marker": "o"},
|
||
"after": {"color": PALETTE[1], "marker": "s"},
|
||
}
|
||
|
||
|
||
def style(name):
|
||
index = ORDER.index(name) if name in ORDER else len(ORDER)
|
||
return {"color": PALETTE[index], "marker": MARKERS[index]}
|
||
|
||
|
||
def load(paths):
|
||
groups = collections.defaultdict(lambda: collections.defaultdict(list))
|
||
for path in paths:
|
||
with Path(path).open() as handle:
|
||
for row in csv.DictReader(line for line in handle if not line.startswith("#")):
|
||
if row["disturbed"] == "True":
|
||
raise ValueError(f"disturbed run in accepted sample: {path}")
|
||
key = (Path(path).parent.name, row["variant"], row["config"])
|
||
groups[key][int(row["rate"])].append(row)
|
||
for key, rates in groups.items():
|
||
for rate, rows in rates.items():
|
||
if len(rows) < MIN_REPEATS or len({row["repeat"] for row in rows}) != len(rows):
|
||
raise ValueError(f"need at least three distinct repeats: {key}, {rate}")
|
||
return groups
|
||
|
||
|
||
def med(rows, field):
|
||
values = [float(r[field]) for r in rows]
|
||
middle = statistics.median(values)
|
||
return middle, middle - min(values), max(values) - middle
|
||
|
||
|
||
def label(key):
|
||
return key[2] if key[1] == "base" else f"{key[2]} {key[1]}"
|
||
|
||
|
||
def summarize(key, rates):
|
||
series, variant, config = key
|
||
sat = {
|
||
rate: 2 * sum(r["saturated"] == "True" for r in rows) > len(rows)
|
||
for rate, rows in rates.items()
|
||
}
|
||
capacity = max((rate for rate, bad in sat.items() if not bad), default=None)
|
||
peak = max(rates, key=lambda rate: med(rates[rate], "ok_rps")[0])
|
||
top, first = rates[peak], rates[peak][0]
|
||
replicas, workers, pool = (int(first[f]) for f in ("replicas", "workers", "pool"))
|
||
row = {
|
||
"series": series,
|
||
"variant": variant,
|
||
"config": config,
|
||
"rates": " ".join(str(r) for r in sorted(rates)),
|
||
"peak_rate": peak,
|
||
"peak_ok_rps": med(top, "ok_rps")[0],
|
||
"peak_ok_min": min(float(r["ok_rps"]) for r in top),
|
||
"peak_ok_max": max(float(r["ok_rps"]) for r in top),
|
||
"peak_p95_ms": med(top, "p95_ms")[0],
|
||
"peak_failed_ratio": med(top, "failed_ratio")[0],
|
||
"peak_dropped": med(top, "dropped_iterations")[0],
|
||
"cpu_api_limit": round(replicas * float(first["cpus"]) * 100),
|
||
"pool_total": replicas * workers * pool,
|
||
}
|
||
for field in (
|
||
"cpu_api cpu_postgres cpu_lb cpu_k6 pg_active_avg pg_lock_avg pg_idle_tx_avg "
|
||
"pg_idle_avg pg_backends_max lb_qcur_max"
|
||
).split():
|
||
row[field] = round(med(top, field)[0], 2)
|
||
if capacity is not None:
|
||
for field, name in (("ok_rps", "ok_rps"), ("p95_ms", "p95_ms"), ("p99_ms", "p99_ms")):
|
||
row[f"capacity_{name}"] = med(rates[capacity], field)[0]
|
||
row["capacity_rate"] = capacity
|
||
row["capacity_label"] = (
|
||
f"≥{capacity}" if capacity in (min(rates), max(rates)) else str(capacity)
|
||
)
|
||
return row
|
||
|
||
|
||
def direct_label(axis, x, y, text):
|
||
axis.annotate(
|
||
text, (x, y), xytext=(6, 0), textcoords="offset points", va="center", color=INK, fontsize=9
|
||
)
|
||
|
||
|
||
def save(figure, path):
|
||
figure.tight_layout()
|
||
figure.savefig(path, dpi=150, metadata={"Description": STAMP})
|
||
plt.close(figure)
|
||
|
||
|
||
def lines(axis, keys, groups, field, categorical=False):
|
||
for index, key in enumerate(keys):
|
||
rates = sorted(groups[key])
|
||
values = [med(groups[key][rate], field) for rate in rates]
|
||
look = VARIANTS.get(key[1]) or style(key[2])
|
||
if categorical:
|
||
look = {"color": PALETTE[index], "marker": MARKERS[index]}
|
||
axis.errorbar(
|
||
[index] if categorical else rates,
|
||
[v[0] for v in values],
|
||
yerr=[[v[1] for v in values], [v[2] for v in values]],
|
||
capsize=3,
|
||
label=label(key),
|
||
markeredgecolor=SURFACE,
|
||
markeredgewidth=1.5,
|
||
**look,
|
||
)
|
||
if categorical:
|
||
axis.set_xticks(range(len(keys)), [label(k) for k in keys], rotation=25, ha="right")
|
||
|
||
|
||
def throughput(series, keys, groups, out):
|
||
figure, axes = plt.subplots(1, 3, figsize=(17, 4.8))
|
||
top = max(rate for key in keys for rate in groups[key])
|
||
categorical = len({rate for key in keys for rate in groups[key]}) == 1
|
||
if not categorical:
|
||
axes[0].plot(
|
||
[0, top], [0, top], color=MUTED, linestyle="--", linewidth=1, label="RPS = нагрузка"
|
||
)
|
||
lines(axes[0], keys, groups, "ok_rps", categorical)
|
||
axes[0].set_title("Успешные RPS")
|
||
for axis, field, title in ((axes[1], "p95_ms", "p95, мс"), (axes[2], "p99_ms", "p99, мс")):
|
||
lines(axis, keys, groups, field, categorical)
|
||
if not categorical:
|
||
axis.set_yscale("log")
|
||
axis.set_title(title)
|
||
for axis in axes:
|
||
axis.set_xlabel(
|
||
f"Вариант при {top} запросах/с" if categorical else "Предложенная нагрузка, запросов/с"
|
||
)
|
||
if not categorical:
|
||
axis.legend(loc="best", fontsize=9)
|
||
figure.suptitle(f"{series}: медиана повторов, усы — min–max", color=INK)
|
||
save(figure, out / f"{series}-throughput.png")
|
||
|
||
|
||
def cpu(series, keys, groups, out):
|
||
figure, axes = plt.subplots(1, len(keys), figsize=(5 * len(keys), 4.2), squeeze=False)
|
||
for axis, key in zip(axes[0], keys, strict=True):
|
||
rates = sorted(groups[key])
|
||
first = groups[key][rates[0]][0]
|
||
limit = int(first["replicas"]) * float(first["cpus"]) * 100
|
||
for index, (field, name) in enumerate(COMPONENTS):
|
||
values = [med(groups[key][rate], f"cpu_{field}") for rate in rates]
|
||
axis.errorbar(
|
||
rates,
|
||
[v[0] for v in values],
|
||
yerr=[[v[1] for v in values], [v[2] for v in values]],
|
||
capsize=3,
|
||
color=PALETTE[index],
|
||
marker=MARKERS[index],
|
||
label=name,
|
||
)
|
||
axis.axhline(limit, color=PALETTE[0], linestyle=":", linewidth=1.2, label="лимит API")
|
||
axis.axhline(200, color=PALETTE[1], linestyle=":", linewidth=1.2, label="лимит PostgreSQL")
|
||
axis.set_title(f"CPU, % ядра — {label(key)}")
|
||
axis.set_xlabel("Предложенная нагрузка, запросов/с")
|
||
axis.legend(fontsize=8, loc="best")
|
||
figure.suptitle("Медиана CPU по повторам, усы — min–max", color=INK)
|
||
save(figure, out / f"{series}-cpu.png")
|
||
|
||
|
||
def usl(points):
|
||
base = points[0][1] / points[0][0]
|
||
pairs = [(n - 1, n * (n - 1), n * base / x - 1) for n, x in points]
|
||
a, b, c = (sum(p[i] * p[j] for p in pairs) for i, j in ((0, 0), (0, 1), (1, 1)))
|
||
d, e = (sum(p[i] * p[2] for p in pairs) for i in (0, 1))
|
||
det = a * c - b * b
|
||
sigma, kappa = (d * c - b * e) / det, (a * e - b * d) / det
|
||
peak = ((1 - sigma) / kappa) ** 0.5 if kappa > 0 else None
|
||
return base, sigma, kappa, peak
|
||
|
||
|
||
def scaling(rows, out):
|
||
by = {(r["series"], r["config"]): r for r in rows if r["variant"] == "base"}
|
||
growth = [
|
||
by.get(("series-b", c)) or by.get(("series-a", c)) for c in ("1x1", "2x1", "4x1", "8x1")
|
||
]
|
||
growth = [r for r in growth if r]
|
||
if not growth:
|
||
return
|
||
figure, axis = plt.subplots(figsize=(7.5, 4.8))
|
||
cpus = [r["cpu_api_limit"] / 100 for r in growth]
|
||
unit = growth[0]["peak_ok_rps"] / cpus[0]
|
||
axis.plot(cpus, [unit * c for c in cpus], color=MUTED, linestyle="--", linewidth=1)
|
||
axis.annotate("линейный рост", (cpus[-1], unit * cpus[-1]), color=MUTED, fontsize=9)
|
||
fit = None
|
||
if len(growth) >= USL_POINTS:
|
||
fit = usl([(c, r["peak_ok_rps"]) for c, r in zip(cpus, growth, strict=True)])
|
||
base, sigma, kappa, _ = fit
|
||
grid = [1 + i * (cpus[-1] - 1) / 60 for i in range(61)]
|
||
model = [base * n / (1 + sigma * (n - 1) + kappa * n * (n - 1)) for n in grid]
|
||
axis.plot(
|
||
grid, model, color=PALETTE[6], linewidth=1.5, label=f"USL: σ={sigma:.3f}, κ={kappa:.4f}"
|
||
)
|
||
axis.errorbar(
|
||
cpus,
|
||
[r["peak_ok_rps"] for r in growth],
|
||
yerr=[
|
||
[r["peak_ok_rps"] - r["peak_ok_min"] for r in growth],
|
||
[r["peak_ok_max"] - r["peak_ok_rps"] for r in growth],
|
||
],
|
||
color=PALETTE[0],
|
||
marker="o",
|
||
capsize=3,
|
||
label="серия B: N × 1 CPU",
|
||
)
|
||
for r, c in zip(growth, cpus, strict=True):
|
||
direct_label(axis, c, r["peak_ok_rps"], r["config"])
|
||
for config in ("2x2", "1x4"):
|
||
r = by.get(("series-a", config))
|
||
if r:
|
||
look = style(config)
|
||
axis.errorbar(
|
||
4,
|
||
r["peak_ok_rps"],
|
||
yerr=[[r["peak_ok_rps"] - r["peak_ok_min"]], [r["peak_ok_max"] - r["peak_ok_rps"]]],
|
||
capsize=3,
|
||
linestyle="none",
|
||
label=f"серия A: {config}",
|
||
**look,
|
||
)
|
||
after = [r for r in rows if r["variant"] == "after"]
|
||
for r in after:
|
||
width = r["cpu_api_limit"] / 100
|
||
axis.errorbar(
|
||
width,
|
||
r["peak_ok_rps"],
|
||
yerr=[[r["peak_ok_rps"] - r["peak_ok_min"]], [r["peak_ok_max"] - r["peak_ok_rps"]]],
|
||
capsize=3,
|
||
linestyle="none",
|
||
label=f"{r['config']} после улучшения",
|
||
**VARIANTS["after"],
|
||
)
|
||
axis.set_xlabel("Суммарный CPU реплик API")
|
||
axis.set_ylabel("Пик успешных RPS")
|
||
axis.set_title("Пропускная способность от ресурсов API")
|
||
axis.set_xticks(cpus)
|
||
axis.legend(loc="best", fontsize=9)
|
||
save(figure, out / "scaling.png")
|
||
return fit
|
||
|
||
|
||
def utilisation(rows, out):
|
||
picked = [r for r in rows if r["variant"] == "base" and r["series"] != "compare"]
|
||
seen, ordered = set(), []
|
||
for config in ORDER:
|
||
for r in picked:
|
||
if r["config"] == config and config not in seen:
|
||
seen.add(config)
|
||
ordered.append(r)
|
||
if not ordered:
|
||
return
|
||
metrics = (
|
||
("CPU API, % лимита", lambda r: 100 * r["cpu_api"] / r["cpu_api_limit"]),
|
||
("CPU PostgreSQL, % лимита", lambda r: r["cpu_postgres"] / 2),
|
||
("CPU HAProxy, % лимита", lambda r: r["cpu_lb"]),
|
||
("ждут блокировку, % пула", lambda r: 100 * r["pg_lock_avg"] / r["pool_total"]),
|
||
)
|
||
figure, axis = plt.subplots(figsize=(10, 4.8))
|
||
width = 0.8 / len(metrics)
|
||
for index, (name, value) in enumerate(metrics):
|
||
xs = [i + (index - 1.5) * width for i in range(len(ordered))]
|
||
axis.bar(
|
||
xs,
|
||
[value(r) for r in ordered],
|
||
width=width - 0.02,
|
||
color=PALETTE[index],
|
||
label=name,
|
||
edgecolor=SURFACE,
|
||
)
|
||
axis.set_xticks(range(len(ordered)), [r["config"] for r in ordered])
|
||
axis.axhline(100, color=MUTED, linestyle=":", linewidth=1)
|
||
axis.set_ylabel("%")
|
||
axis.set_ylim(0, 115)
|
||
axis.set_title("Загрузка звеньев в точке пика успешных RPS", pad=34)
|
||
axis.legend(fontsize=9, ncols=4, loc="lower left", bbox_to_anchor=(0, 1.0))
|
||
save(figure, out / "utilisation.png")
|
||
|
||
|
||
def write(rows, out, fit):
|
||
with (out / "capacity.csv").open("w", newline="") as handle:
|
||
handle.write(f"# {STAMP}\n")
|
||
writer = csv.DictWriter(handle, fieldnames=SUMMARY, extrasaction="ignore")
|
||
writer.writeheader()
|
||
writer.writerows(rows)
|
||
head = (
|
||
"| серия | вариант | конфигурация | потолок по SLO, rps | p95 там, мс | пик успешных RPS "
|
||
"[min–max] | при нагрузке | CPU API / лимит | CPU PG | CPU LB | ждут блокировку / пул |"
|
||
)
|
||
lines_ = [head, "|" + " --- |" * 11]
|
||
for r in rows:
|
||
lines_.append(
|
||
f"| {r['series']} | {r['variant']} | {r['config']} | {r.get('capacity_label', '—')} | "
|
||
f"{r.get('capacity_p95_ms', '—')} | {r['peak_ok_rps']:.0f} "
|
||
f"[{r['peak_ok_min']:.0f}–{r['peak_ok_max']:.0f}] | {r['peak_rate']} | "
|
||
f"{r['cpu_api']:.0f} / {r['cpu_api_limit']} | {r['cpu_postgres']:.0f} | "
|
||
f"{r['cpu_lb']:.0f} | {r['pg_lock_avg']:.1f} / {r['pool_total']} |"
|
||
)
|
||
if fit:
|
||
_, sigma, kappa, peak = fit
|
||
best = f"{peak:.1f}" if peak else "—"
|
||
lines_.append(
|
||
f"\nОписательная USL-аппроксимация серии B: σ = {sigma:.4f}, κ = {kappa:.5f}, "
|
||
f"модельный N* = {best} CPU. Это не измеренный оптимум и не доказательство причины."
|
||
)
|
||
(out / "capacity.md").write_text(STAMP + "\n\n" + "\n".join(lines_) + "\n")
|
||
print("\n".join(lines_))
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description="Графики и сводка трека 1 из summary.csv")
|
||
parser.add_argument("summaries", nargs="+")
|
||
parser.add_argument("--out", required=True)
|
||
args = parser.parse_args()
|
||
out = Path(args.out)
|
||
out.mkdir(parents=True, exist_ok=True)
|
||
groups = load(args.summaries)
|
||
rank = {name: index for index, name in enumerate(ORDER)}
|
||
keys = sorted(groups, key=lambda k: (k[0], k[1] != "before", rank.get(k[2], 99)))
|
||
rows = [summarize(key, groups[key]) for key in keys]
|
||
for series in dict.fromkeys(k[0] for k in keys):
|
||
chosen = [k for k in keys if k[0] == series]
|
||
throughput(series, chosen, groups, out)
|
||
cpu(series, chosen, groups, out)
|
||
fit = scaling(rows, out)
|
||
utilisation(rows, out)
|
||
write(rows, out, fit)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|