413 lines
17 KiB
Python
413 lines
17 KiB
Python
"""Графики серий трека 1: SVG на стандартной библиотеке.
|
||
|
||
uv run python bench/plots.py --out scaling scaling-a scaling-b scaling-db
|
||
→ evidence/plots/<out>/: rstar-by-config.svg, rstar-by-replicas.svg,
|
||
summary.md (R*, эффективность, точка насыщения)
|
||
|
||
`run.py` после серии зовёт `render_label()`: в каталоге метки —
|
||
`p95-by-rate.svg` (p95 от поданной интенсивности по ячейкам) и
|
||
`cpu-by-component.svg` (CPU компонентов на ступени R*). Числа берутся из
|
||
`searches.csv` и `runs.csv` меток как есть.
|
||
"""
|
||
|
||
import argparse
|
||
import csv
|
||
import html
|
||
import math
|
||
import re
|
||
import statistics
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
SCALING = ROOT / "evidence" / "scaling"
|
||
PLOTS = ROOT / "evidence" / "plots"
|
||
COLORS = (
|
||
"#2f6fbf",
|
||
"#d0752a",
|
||
"#3c9a5f",
|
||
"#b8404a",
|
||
"#7a5cb5",
|
||
"#8c6d3f",
|
||
"#c25f9c",
|
||
"#5f8f9c",
|
||
)
|
||
COMPONENTS = (
|
||
("api_cpu_cores", "API (сумма реплик)", "#2f6fbf"),
|
||
("db_cpu_cores", "PostgreSQL", "#b8404a"),
|
||
("lb_cpu_cores", "HAProxy", "#3c9a5f"),
|
||
("cache_cpu_cores", "Valkey", "#7a5cb5"),
|
||
("gen_cpu_cores", "генератор", "#8c6d3f"),
|
||
)
|
||
CONFIG_RE = re.compile(r"(\d+)x([\d.]+)(?:-pg([\d.]+))?")
|
||
|
||
|
||
def read_csv(path: Path) -> list[dict]:
|
||
if not path.exists():
|
||
return []
|
||
with path.open(encoding="utf-8") as handle:
|
||
return list(csv.DictReader(handle))
|
||
|
||
|
||
def num(value) -> float | None:
|
||
try:
|
||
return float(value)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def nice_step(raw: float) -> float:
|
||
"""Круглый шаг сетки не меньше raw: 1, 2, 2,5, 5 или 10 × 10^k."""
|
||
magnitude = 10 ** math.floor(math.log10(raw))
|
||
return next(m * magnitude for m in (1, 2, 2.5, 5, 10) if m * magnitude >= raw)
|
||
|
||
|
||
def nice_ticks(top: float, count: int = 6) -> list[float]:
|
||
step = nice_step(top / count) if top > 0 else 1.0
|
||
return [round(step * i, 6) for i in range(int(top / step) + 1)]
|
||
|
||
|
||
class Svg:
|
||
"""Поле графика с осями: x и y в единицах данных."""
|
||
|
||
def __init__(self, width: int, height: int, title: str) -> None:
|
||
self.width, self.height = width, height
|
||
# справа — место под легенду: имена меток длинные
|
||
self.left, self.right, self.top, self.bottom = 70, 300, 40, 50
|
||
self.parts = [
|
||
f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" '
|
||
f'viewBox="0 0 {width} {height}" font-family="sans-serif" font-size="12">',
|
||
f'<rect width="{width}" height="{height}" fill="white"/>',
|
||
f'<text x="{width / 2}" y="22" text-anchor="middle" font-size="14">'
|
||
f"{html.escape(title)}</text>",
|
||
]
|
||
self.x0 = self.x1 = self.y0 = self.y1 = 0.0
|
||
self.ystep = 1.0
|
||
self.legend_row = 0
|
||
|
||
def scale(self, x0: float, x1: float, y0: float, y1: float) -> None:
|
||
"""Ось y — от y0 до круглого значения не меньше y1 (шаг сетки
|
||
1, 2, 2,5 или 5 × 10^k, пять–шесть делений)."""
|
||
self.ystep = nice_step(max(y1 - y0, 1e-9) / 5)
|
||
top = y0 + self.ystep * math.ceil(max(y1 - y0, 1e-9) / self.ystep)
|
||
self.x0, self.x1, self.y0, self.y1 = (
|
||
x0,
|
||
x1 if x1 > x0 else x0 + 1,
|
||
y0,
|
||
top,
|
||
)
|
||
|
||
def px(self, x: float) -> float:
|
||
span = self.width - self.left - self.right
|
||
return self.left + (x - self.x0) / (self.x1 - self.x0) * span
|
||
|
||
def py(self, y: float) -> float:
|
||
span = self.height - self.top - self.bottom
|
||
return self.height - self.bottom - (y - self.y0) / (self.y1 - self.y0) * span
|
||
|
||
def axes(self, xlabel: str, ylabel: str, xticks: list[tuple[float, str]]) -> None:
|
||
bottom, left = self.height - self.bottom, self.left
|
||
right = self.width - self.right
|
||
self.parts.append(
|
||
f'<line x1="{left}" y1="{bottom}" x2="{right}" y2="{bottom}" stroke="#333"/>'
|
||
)
|
||
self.parts.append(
|
||
f'<line x1="{left}" y1="{self.top}" x2="{left}" y2="{bottom}" stroke="#333"/>'
|
||
)
|
||
for x, text in xticks:
|
||
self.parts.append(
|
||
f'<text x="{self.px(x):.1f}" y="{bottom + 16}" text-anchor="middle">{html.escape(text)}</text>'
|
||
)
|
||
for index in range(round((self.y1 - self.y0) / self.ystep) + 1):
|
||
value = round(self.y0 + self.ystep * index, 6)
|
||
y = self.py(value)
|
||
self.parts.append(
|
||
f'<line x1="{left}" y1="{y:.1f}" x2="{right}" y2="{y:.1f}" stroke="#eee"/>'
|
||
)
|
||
self.parts.append(
|
||
f'<text x="{left - 6}" y="{y + 4:.1f}" text-anchor="end">{value:g}</text>'
|
||
)
|
||
self.parts.append(
|
||
f'<text x="{(left + right) / 2}" y="{self.height - 12}" text-anchor="middle">{html.escape(xlabel)}</text>'
|
||
)
|
||
self.parts.append(
|
||
f'<text x="16" y="{(self.top + bottom) / 2}" text-anchor="middle" '
|
||
f'transform="rotate(-90 16 {(self.top + bottom) / 2})">{html.escape(ylabel)}</text>'
|
||
)
|
||
|
||
def line(
|
||
self, points: list[tuple[float, float]], color: str, dash: str = ""
|
||
) -> None:
|
||
if not points:
|
||
return
|
||
path = " ".join(f"{self.px(x):.1f},{self.py(y):.1f}" for x, y in points)
|
||
extra = f' stroke-dasharray="{dash}"' if dash else ""
|
||
self.parts.append(
|
||
f'<polyline points="{path}" fill="none" stroke="{color}" stroke-width="2"{extra}/>'
|
||
)
|
||
|
||
def dot(self, x: float, y: float, color: str, hollow: bool = False) -> None:
|
||
fill = "white" if hollow else color
|
||
self.parts.append(
|
||
f'<circle cx="{self.px(x):.1f}" cy="{self.py(y):.1f}" r="3.5" fill="{fill}" stroke="{color}"/>'
|
||
)
|
||
|
||
def whisker(self, x: float, lo: float, hi: float, color: str) -> None:
|
||
self.parts.append(
|
||
f'<line x1="{self.px(x):.1f}" y1="{self.py(lo):.1f}" x2="{self.px(x):.1f}" '
|
||
f'y2="{self.py(hi):.1f}" stroke="{color}" stroke-width="1.5"/>'
|
||
)
|
||
|
||
def bar(self, x: float, width: float, y0: float, y1: float, color: str) -> None:
|
||
left = self.px(x - width / 2)
|
||
right = self.px(x + width / 2)
|
||
self.parts.append(
|
||
f'<rect x="{left:.1f}" y="{self.py(y1):.1f}" width="{right - left:.1f}" '
|
||
f'height="{self.py(y0) - self.py(y1):.1f}" fill="{color}" opacity="0.85"/>'
|
||
)
|
||
|
||
def legend(self, text: str, color: str, dash: str = "") -> None:
|
||
x = self.width - self.right + 14
|
||
y = self.top + 14 + 18 * self.legend_row
|
||
extra = f' stroke-dasharray="{dash}"' if dash else ""
|
||
self.parts.append(
|
||
f'<line x1="{x}" y1="{y - 4}" x2="{x + 18}" y2="{y - 4}" stroke="{color}" stroke-width="3"{extra}/>'
|
||
)
|
||
self.parts.append(f'<text x="{x + 24}" y="{y}">{html.escape(text)}</text>')
|
||
self.legend_row += 1
|
||
|
||
def save(self, path: Path) -> None:
|
||
path.write_text("\n".join([*self.parts, "</svg>"]) + "\n", encoding="utf-8")
|
||
|
||
|
||
def render_label(folder: Path) -> None:
|
||
"""Графики одной метки серии: p95 от интенсивности, CPU на ступени R*."""
|
||
runs = read_csv(folder / "runs.csv")
|
||
searches = read_csv(folder / "searches.csv")
|
||
if not runs:
|
||
return
|
||
configs = list(dict.fromkeys(row["config"] for row in runs))
|
||
rates = [num(r["rate"]) or 0 for r in runs]
|
||
p95s = [min(num(r["p95_ms"]) or 0, 1000) for r in runs]
|
||
svg = Svg(1000, 440, f"p95 успешных от поданной интенсивности: {folder.name}")
|
||
svg.scale(0, max(rates) * 1.05, 0, max(max(p95s) * 1.1, 250))
|
||
ticks = [(x, f"{x:g}") for x in nice_ticks(max(rates))]
|
||
svg.axes("запросов в секунду (по расписанию)", "p95, мс (≤ 1000)", ticks)
|
||
svg.line([(0, 200), (max(rates) * 1.05, 200)], "#999", "4 4")
|
||
for index, config in enumerate(configs):
|
||
color = COLORS[index % len(COLORS)]
|
||
cells = sorted(
|
||
{(r["cell"], r["repeat"]) for r in runs if r["config"] == config}
|
||
)
|
||
for cell, _ in cells:
|
||
points = sorted(
|
||
(num(r["rate"]) or 0, min(num(r["p95_ms"]) or 1000, 1000))
|
||
for r in runs
|
||
if r["cell"] == cell
|
||
)
|
||
svg.line(points, color)
|
||
for r in runs:
|
||
if r["cell"] == cell:
|
||
svg.dot(
|
||
num(r["rate"]) or 0,
|
||
min(num(r["p95_ms"]) or 1000, 1000),
|
||
color,
|
||
hollow=r["slo_pass"] != "True",
|
||
)
|
||
svg.legend(config, color)
|
||
svg.legend("SLO 200 мс", "#999", "4 4")
|
||
svg.save(folder / "p95-by-rate.svg")
|
||
|
||
# CPU компонентов на ступени R* каждой ячейки (медиана по повторам)
|
||
stacks = []
|
||
for config in configs:
|
||
values = {key: [] for key, _, _ in COMPONENTS}
|
||
for search in [s for s in searches if s["config"] == config]:
|
||
star = num(search["r_star"])
|
||
row = next(
|
||
(
|
||
r
|
||
for r in runs
|
||
if r["cell"] == search["cell"]
|
||
and r["phase"] != "fixed"
|
||
and num(r["rate"]) == star
|
||
),
|
||
None,
|
||
)
|
||
if row is None:
|
||
continue
|
||
for key, _, _ in COMPONENTS:
|
||
if num(row.get(key)) is not None:
|
||
values[key].append(num(row[key]))
|
||
stacks.append(
|
||
(config, {k: statistics.median(v) if v else 0 for k, v in values.items()})
|
||
)
|
||
if not stacks:
|
||
return
|
||
top = max(sum(s.values()) for _, s in stacks) or 1
|
||
svg = Svg(
|
||
1000, 440, f"CPU компонентов на ступени R* (медиана), ядер: {folder.name}"
|
||
)
|
||
svg.scale(0, len(stacks) + 1, 0, top * 1.15)
|
||
svg.axes("конфигурация", "ядер", [(i + 1, c) for i, (c, _) in enumerate(stacks)])
|
||
for index, (_, stack) in enumerate(stacks):
|
||
base = 0.0
|
||
for key, _, color in COMPONENTS:
|
||
svg.bar(index + 1, 0.5, base, base + stack[key], color)
|
||
base += stack[key]
|
||
for _, name, color in COMPONENTS:
|
||
svg.legend(name, color)
|
||
svg.save(folder / "cpu-by-component.svg")
|
||
|
||
|
||
def label_rows(label: str) -> list[dict]:
|
||
rows = read_csv(SCALING / label / "searches.csv")
|
||
for row in rows:
|
||
row["label"] = label
|
||
return rows
|
||
|
||
|
||
def summarize(rows: list[dict]) -> dict[tuple[str, str], dict]:
|
||
"""(метка, конфигурация) → медиана, min, max R*, реплик, CPU."""
|
||
out: dict[tuple[str, str], dict] = {}
|
||
for row in rows:
|
||
key = (row["label"], row["config"])
|
||
entry = out.setdefault(
|
||
key,
|
||
{
|
||
"values": [],
|
||
"replicas": int(row["replicas"]),
|
||
"api_cpus": num(row["api_cpus"]),
|
||
"db_cpus": num(row["db_cpus"]),
|
||
"signals": {},
|
||
},
|
||
)
|
||
if num(row["r_star"]) is not None:
|
||
entry["values"].append(num(row["r_star"]))
|
||
for signal in (row["signals_at_fail"] or "").split(";"):
|
||
if signal:
|
||
entry["signals"][signal] = entry["signals"].get(signal, 0) + 1
|
||
for entry in out.values():
|
||
values = entry["values"] or [0]
|
||
entry["median"] = statistics.median(values)
|
||
entry["min"], entry["max"] = min(values), max(values)
|
||
return out
|
||
|
||
|
||
def saturation(points: list[tuple[int, float]]) -> int | None:
|
||
"""Наименьшее k, после которого следующая реплика даёт < +10 % R*."""
|
||
points = sorted(points)
|
||
for (k, r), (_, r_next) in zip(points, points[1:], strict=False):
|
||
if r_next < 1.10 * r:
|
||
return k
|
||
return None
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
parser.add_argument("--out", required=True)
|
||
parser.add_argument("labels", nargs="+")
|
||
args = parser.parse_args()
|
||
if not re.fullmatch(r"[A-Za-z0-9._-]{1,48}", args.out):
|
||
raise SystemExit("--out must be a simple name")
|
||
rows = [row for label in args.labels for row in label_rows(label)]
|
||
if not rows:
|
||
raise SystemExit("no searches.csv in the given labels")
|
||
stats = summarize(rows)
|
||
folder = PLOTS / args.out
|
||
folder.mkdir(parents=True, exist_ok=True)
|
||
|
||
# R* по конфигурациям всех меток: медиана и min–max
|
||
items = list(stats.items())
|
||
top = max(entry["max"] for _, entry in items) or 1
|
||
svg = Svg(
|
||
max(1000, 90 * len(items) + 380), 460, "R* по конфигурациям: медиана и min–max"
|
||
)
|
||
svg.scale(0, len(items) + 1, 0, top * 1.15)
|
||
svg.axes(
|
||
"метка / конфигурация",
|
||
"R*, запросов/с",
|
||
[(i + 1, cfg) for i, ((_, cfg), _) in enumerate(items)],
|
||
)
|
||
labels = list(dict.fromkeys(label for (label, _), _ in items))
|
||
for index, ((label, _), entry) in enumerate(items):
|
||
color = COLORS[labels.index(label) % len(COLORS)]
|
||
svg.bar(index + 1, 0.55, 0, entry["median"], color)
|
||
svg.whisker(index + 1, entry["min"], entry["max"], "#222")
|
||
for label in labels:
|
||
svg.legend(label, COLORS[labels.index(label) % len(COLORS)])
|
||
svg.save(folder / "rstar-by-config.svg")
|
||
|
||
# R* от числа реплик по 1 CPU — по меткам, где такие конфигурации есть
|
||
svg = Svg(
|
||
860, 440, "R* от числа реплик (по 1 CPU): медиана, min–max, линейный рост"
|
||
)
|
||
curves = {}
|
||
for (label, config), entry in items:
|
||
match = CONFIG_RE.fullmatch(config)
|
||
if match and float(match.group(2)) == 1.0:
|
||
curves.setdefault((label, match.group(3) or "2"), []).append(
|
||
(entry["replicas"], entry)
|
||
)
|
||
if curves:
|
||
max_k = max(k for points in curves.values() for k, _ in points)
|
||
# линия линейного роста от 1 × 1 тоже должна поместиться
|
||
ideal = [
|
||
e["median"] * max_k
|
||
for points in curves.values()
|
||
for k, e in points
|
||
if k == 1
|
||
]
|
||
svg.scale(0, max_k + 0.5, 0, max([top, *ideal]) * 1.05)
|
||
svg.axes("реплик", "R*, запросов/с", [(k, str(k)) for k in range(1, max_k + 1)])
|
||
for index, ((label, pg), points) in enumerate(sorted(curves.items())):
|
||
color = COLORS[index % len(COLORS)]
|
||
points.sort(key=lambda p: p[0])
|
||
svg.line([(k, e["median"]) for k, e in points], color)
|
||
for k, entry in points:
|
||
svg.dot(k, entry["median"], color)
|
||
svg.whisker(k, entry["min"], entry["max"], color)
|
||
one = next((e["median"] for k, e in points if k == 1), None)
|
||
if one:
|
||
svg.line([(1, one), (max_k, one * max_k)], color, "3 4")
|
||
svg.legend(f"{label}, PostgreSQL {pg} CPU", color)
|
||
svg.save(folder / "rstar-by-replicas.svg")
|
||
|
||
lines = [
|
||
f"# Графики трека 1: {args.out}",
|
||
"",
|
||
f"Метки: {', '.join(args.labels)}. Числа — `searches.csv` меток.",
|
||
"",
|
||
"| Метка | Конфигурация | Реплик | CPU реплики | PostgreSQL CPU | R* медиана | min–max | Эффективность к 1 × 1 метки | Признаки на первой нарушившей |",
|
||
"|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for (label, config), entry in items:
|
||
one = stats.get((label, "1x1"))
|
||
eff = (
|
||
f"{entry['median'] / (entry['replicas'] * one['median']):.2f}"
|
||
if one and one["median"] and config != "1x1"
|
||
else "—"
|
||
)
|
||
signals = (
|
||
", ".join(f"{k} ×{v}" for k, v in sorted(entry["signals"].items())) or "—"
|
||
)
|
||
lines.append(
|
||
f"| {label} | {config} | {entry['replicas']} | {entry['api_cpus']:g} | "
|
||
f"{entry['db_cpus']:g} | {entry['median']:g} | {entry['min']:g}–{entry['max']:g} | "
|
||
f"{eff} | {signals} |"
|
||
)
|
||
lines += ["", "## Точка насыщения", ""]
|
||
for (label, pg), points in sorted(curves.items()):
|
||
k = saturation([(k, e["median"]) for k, e in points])
|
||
verdict = (
|
||
f"после {k} реплик(и) следующая даёт меньше +10 % R*"
|
||
if k
|
||
else "в пределах снятых конфигураций рост не останавливается (≥ +10 % на реплику)"
|
||
)
|
||
lines.append(f"- {label}, PostgreSQL {pg} CPU: {verdict}.")
|
||
(folder / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
print(f"written: {folder.relative_to(ROOT)}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|