Files
mtusi2026lab3/bench/track1/analyze.py
T
Alexey BodanyukandClaude Opus 5.5 b9f1ad58f9 bench: трек 1 — профили py-spy до/после, серия E (потолок нового пути), графики
Профиль при 240 req/s: доля psycopg 52.7 % -> 19.0 %, активных сэмплов
3889 -> 2026. Серия E: function в 2x1.0 CPU — потолок 552 (530–552)
успешных/с, упор в CPU API; в 3x1.0 / БД 0.5 — ~420–460, упор в БД.
analyze.py: линии без заливки, цвет = конфигурация, пунктир = multi,
подпись — максимум серии. REPORT: раздел трека 1 и ограничения.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 17:33:46 +03:00

221 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Track 1 analysis: evidence/track1/<series>/<label>/results.csv -> summary.md + SVG charts.
python bench/track1/analyze.py # all series found under evidence/track1
Per configuration and offered rate: median over repeats (and min..max) of SUCCESSFUL req/s
(only 201 responses count; errors and dropped iterations are failed demand), p95 of successful
requests, CPU of every component as a share of ITS OWN limit. Capacity of a configuration =
per repeat the highest successful req/s over the sweep, then median and range over repeats.
SLO capacity = the highest offered rate that was served: ok >= 97 % of offered, p95 <= 500 ms.
Bottleneck = the component whose CPU is >= 90 % of its limit at the capacity step.
Standard library only; charts are plain SVG (one y-axis, <= 4 series, legend + labels, hover titles).
"""
from __future__ import annotations
import csv
import re
import statistics
from collections import defaultdict
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
EVIDENCE = ROOT / "evidence" / "track1"
SERIES_COLORS = [("#2a78d6", "#3987e5"), ("#eb6834", "#d95926"), ("#1baf7a", "#199e70"), ("#eda100", "#c98500")]
def num(value):
if value in (None, "", "None"):
return None
return float(value)
CONFIG_RE = re.compile(r"n(?P<n>\d+)-api(?P<api>[\d.]+)-pg(?P<pg>[\d.]+)-pool(?P<pool>\d+)(?:-message_write_path_(?P<path>\w+))?")
def label_of(config: str) -> str:
m = CONFIG_RE.fullmatch(config)
if not m:
return config
text = f"{m['n']} × {m['api']} CPU API, БД {m['pg']} CPU"
if m["path"]:
text += f", запись {m['path']}"
return text
def load(path: Path) -> list[dict]:
with path.open(encoding="utf-8") as f:
return list(csv.DictReader(f))
def aggregate(rows: list[dict]) -> dict:
by_cfg: dict[str, dict] = defaultdict(lambda: defaultdict(list))
for r in rows:
by_cfg[r["config"]][int(r["rate"])].append(r)
result = {}
for cfg, by_rate in by_cfg.items():
points = []
for rate in sorted(by_rate):
rs = by_rate[rate]
ok = [num(r["ok_rps"]) for r in rs]
p95 = [num(r["p95_ms"]) for r in rs if num(r["p95_ms"]) is not None]
points.append({"rate": rate, "ok": statistics.median(ok), "lo": min(ok), "hi": max(ok), "n": len(rs),
"p95": statistics.median(p95) if p95 else None,
"err": statistics.median(num(r["error_ratio"]) for r in rs),
"dropped": statistics.median(num(r["dropped"]) for r in rs),
"api_cpu": statistics.median(num(r["api_cpu_pct_total"]) or 0 for r in rs),
"pg_cpu": statistics.median(num(r["pg_cpu_pct"]) or 0 for r in rs),
"lb_cpu": statistics.median(num(r["lb_cpu_pct"]) or 0 for r in rs),
"k6_cpu": statistics.median(num(r["k6_cpu_pct"]) or 0 for r in rs),
"pool_wait": statistics.median(num(r["pool_wait_ms_avg"]) or 0 for r in rs),
"stmts": statistics.median(num(r["statements_per_ok"]) or 0 for r in rs),
"commits": statistics.median(num(r["pg_commits_per_ok"]) or 0 for r in rs),
"chkdown": sum(int(num(r["lb_chkdown"]) or 0) for r in rs),
"api_limit": num(rs[0].get("api_cpus_limit_total")), "pg_limit": num(rs[0].get("pg_cpus_limit")),
"replicas": num(rs[0].get("replicas_running"))})
repeats = sorted({int(r["repeat"]) for rr in by_rate.values() for r in rr})
caps, slo = [], []
for rep in repeats:
rr = [r for rate_rows in by_rate.values() for r in rate_rows if int(r["repeat"]) == rep]
caps.append(max(num(r["ok_rps"]) for r in rr))
served = [int(r["rate"]) for r in rr if num(r["ok_rps"]) >= 0.97 * int(r["rate"])
and (num(r["p95_ms"]) or 1e9) <= 500]
slo.append(max(served) if served else 0)
best = max(points, key=lambda p: p["ok"])
bottleneck = []
if best["api_limit"] and best["api_cpu"] >= 90 * best["api_limit"]:
bottleneck.append("API CPU")
elif best["replicas"] and best["api_cpu"] >= 90 * best["replicas"] and best["api_limit"] > best["replicas"]:
# One uvicorn process per replica runs Python bytecode on ONE core (GIL), whatever the quota.
bottleneck.append("API: 1 ядро на процесс (GIL), квота не используется")
if best["pg_limit"] and best["pg_cpu"] >= 90 * best["pg_limit"]:
bottleneck.append("PostgreSQL CPU")
if best["lb_cpu"] >= 90:
bottleneck.append("LB CPU")
result[cfg] = {"points": points, "capacity": statistics.median(caps), "cap_lo": min(caps), "cap_hi": max(caps),
"slo": statistics.median(slo), "repeats": len(repeats), "best": best,
"bottleneck": ", ".join(bottleneck) or "нет (ни один CPU не у предела)"}
return result
def svg_chart(title: str, series: dict[str, list[dict]], path: Path, y_key: str = "ok",
y_label: str = "успешных запросов/с (201)", diagonal: bool = True) -> None:
width, height, left, right, top, bottom = 760, 420, 64, 210, 40, 52
xs = [p["rate"] for pts in series.values() for p in pts]
ys = [p.get("hi", p[y_key]) for pts in series.values() for p in pts] + ([max(xs)] if diagonal else [])
x0, x1 = 0, max(xs) * 1.05
y1 = max(ys) * 1.1
sx = lambda v: left + (v - x0) / (x1 - x0) * (width - left - right) # noqa: E731
sy = lambda v: height - bottom - v / y1 * (height - top - bottom) # noqa: E731
out = [f'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 {width} {height}" font-family="system-ui, sans-serif" font-size="12">',
"<style>.bg{fill:#fcfcfb}.ink{fill:#0b0b0b}.ink2{fill:#52514e}.grid{stroke:#e4e3df}.ref{stroke:#b5b3ac}"
+ "".join(f".l{i}{{stroke:{c[0]};fill:none}}.d{i}{{fill:{c[0]};stroke:#fcfcfb}}" for i, c in enumerate(SERIES_COLORS))
+ "@media (prefers-color-scheme: dark){.bg{fill:#1a1a19}.ink{fill:#fff}.ink2{fill:#c3c2b7}.grid{stroke:#34332f}.ref{stroke:#6b6a64}"
+ "".join(f".l{i}{{stroke:{c[1]}}}.d{i}{{fill:{c[1]};stroke:#1a1a19}}" for i, c in enumerate(SERIES_COLORS)) + "}</style>",
f'<rect class="bg" width="{width}" height="{height}"/>',
f'<text class="ink" x="{left}" y="22" font-size="14" font-weight="600">{title}</text>']
step = nice_step(y1)
v = 0.0
while v <= y1:
out.append(f'<line class="grid" x1="{left}" x2="{width - right}" y1="{sy(v):.1f}" y2="{sy(v):.1f}" stroke-width="1"/>')
out.append(f'<text class="ink2" x="{left - 8}" y="{sy(v) + 4:.1f}" text-anchor="end">{v:g}</text>')
v += step
for rate in sorted(set(xs)):
out.append(f'<text class="ink2" x="{sx(rate):.1f}" y="{height - bottom + 18}" text-anchor="middle">{rate}</text>')
out.append(f'<text class="ink2" x="{(left + width - right) / 2}" y="{height - 12}" text-anchor="middle">предложенная нагрузка, запросов/с (open-loop)</text>')
out.append(f'<text class="ink2" transform="translate(16 {(top + height - bottom) / 2}) rotate(-90)" text-anchor="middle">{y_label}</text>')
if diagonal:
out.append(f'<line class="ref" x1="{sx(0):.1f}" y1="{sy(0):.1f}" x2="{sx(max(xs)):.1f}" y2="{sy(max(xs)):.1f}" stroke-width="1"/>')
mid = min(xs) + 0.25 * (max(xs) - min(xs))
out.append(f'<text class="ink2" x="{sx(mid) - 6:.1f}" y="{sy(mid) - 8:.1f}" text-anchor="end">обслужено = предложено</text>')
legend_y = top + 10
# Colour = resource configuration; line style = write path (dashed = old `multi`), so an
# A/B pair of one configuration shares a hue and reads as "before / after".
bases = list(dict.fromkeys(name.split(", запись")[0] for name in series))
for name, pts in series.items():
i = bases.index(name.split(", запись")[0])
dash = ' stroke-dasharray="6 4"' if name.endswith("запись multi") else ""
line_cls, dot_cls = f"l{i}", f"d{i}"
line = " ".join(f"{sx(p['rate']):.1f},{sy(p[y_key]):.1f}" for p in pts)
out.append(f'<polyline class="{line_cls}" points="{line}" stroke-width="2" stroke-linejoin="round" stroke-linecap="round"{dash}/>')
for p in pts:
if "lo" in p and y_key == "ok":
out.append(f'<line class="{line_cls}" x1="{sx(p["rate"]):.1f}" x2="{sx(p["rate"]):.1f}" y1="{sy(p["lo"]):.1f}" y2="{sy(p["hi"]):.1f}" stroke-width="2" opacity="0.5"/>')
tip = (f"{name}: предложено {p['rate']}/с, успешно {p[y_key]:g}/с" + (f" (повторы {p['lo']:g}–{p['hi']:g})" if "lo" in p else "")
+ (f", p95 {p['p95']:.0f} мс" if p.get("p95") else ""))
out.append(f'<circle class="{dot_cls}" cx="{sx(p["rate"]):.1f}" cy="{sy(p[y_key]):.1f}" r="4" stroke-width="2"><title>{tip}</title></circle>')
peak = max(pts, key=lambda q: q[y_key]) # direct label = the ceiling of this series
out.append(f'<text class="ink2" x="{sx(peak["rate"]):.1f}" y="{sy(peak[y_key]) - 10:.1f}" text-anchor="middle">max {peak[y_key]:.0f}</text>')
out.append(f'<line class="{line_cls}" x1="{width - right + 34}" x2="{width - right + 52}" y1="{legend_y}" y2="{legend_y}" stroke-width="2"{dash}/>')
for k, chunk in enumerate(wrap(name, 22)):
out.append(f'<text class="ink" x="{width - right + 58}" y="{legend_y + 4 + 14 * k}">{chunk}</text>')
legend_y += 14 * len(wrap(name, 22)) + 12
out.append("</svg>")
path.write_text("\n".join(out) + "\n", encoding="utf-8")
def wrap(text: str, width: int) -> list[str]:
words, lines, cur = text.split(), [], ""
for w in words:
if len(cur) + len(w) + 1 > width and cur:
lines.append(cur)
cur = w
else:
cur = f"{cur} {w}".strip()
return lines + [cur]
def nice_step(top: float) -> float:
for step in (10, 20, 25, 50, 100, 200, 250, 500, 1000):
if top / step <= 8:
return step
return 2000
def series_section(series_dir: Path) -> list[str]:
lines = []
for label_dir in sorted(p for p in series_dir.iterdir() if (p / "results.csv").exists()):
rows = load(label_dir / "results.csv")
if not rows:
continue # series still running / not started
agg = aggregate(rows)
name = f"{series_dir.name}/{label_dir.name}"
lines += [f"### {name}", "", "| Конфигурация | Повторов | Потолок, успешных/с (медиана, мин–макс) | SLO-ёмкость, предложено/с | "
"p95 на потолке, мс | CPU API / лимит | CPU БД / лимит | CPU LB | CPU k6 | Ожидание пула, мс | SQL-операторов на сообщение | Узкое место |",
"| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"]
for cfg, a in agg.items():
b = a["best"]
lines.append(f"| {label_of(cfg)} | {a['repeats']} | **{a['capacity']:.0f}** ({a['cap_lo']:.0f}–{a['cap_hi']:.0f}) | {a['slo']:.0f} | "
f"{b['p95']:.0f} | {b['api_cpu']:.0f} % / {100 * (b['api_limit'] or 0):.0f} % | "
f"{b['pg_cpu']:.0f} % / {100 * (b['pg_limit'] or 0):.0f} % | {b['lb_cpu']:.0f} % | {b['k6_cpu']:.0f} % | "
f"{b['pool_wait']:.1f} | {b['stmts']:.1f} | {a['bottleneck']} |")
lines += ["", "<details><summary>Все ступени (медианы по повторам)</summary>", "",
"| Конфигурация | Предложено/с | Успешно/с (мин–макс) | p95, мс | Ошибки | Dropped | CPU API | CPU БД | Ожидание пула, мс | DOWN в LB |",
"| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"]
for cfg, a in agg.items():
for p in a["points"]:
lines.append(f"| {label_of(cfg)} | {p['rate']} | {p['ok']:.1f} ({p['lo']:.0f}–{p['hi']:.0f}) | "
f"{(p['p95'] or 0):.0f} | {p['err']:.4f} | {p['dropped']:.0f} | {p['api_cpu']:.0f} % | {p['pg_cpu']:.0f} % | "
f"{p['pool_wait']:.1f} | {p['chkdown']} |")
lines += ["", "</details>", ""]
chart = label_dir / "throughput.svg"
svg_chart(name, {label_of(c): a["points"] for c, a in agg.items()}, chart)
lines += [f"![{name}]({chart.relative_to(EVIDENCE).as_posix()})", ""]
return lines
def main() -> int:
lines = ["# Трек 1 — сводка (сгенерировано bench/track1/analyze.py из results.csv)", "",
"Успешные = только ответы 201 в фазе measure; ошибки и dropped iterations k6 — неудовлетворённый спрос. "
"CPU в % одного ядра (100 % = 1 CPU), медиана выборок docker stats за фазу measure; у API — сумма реплик в каждый "
"момент. Лимиты — фактические из docker inspect.", ""]
for series_dir in sorted(p for p in EVIDENCE.iterdir() if p.is_dir() and p.name[:1].isupper()):
lines += [f"## {series_dir.name}", ""] + series_section(series_dir)
(EVIDENCE / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
print((EVIDENCE / "summary.md").read_text(encoding="utf-8")[:4000])
return 0
if __name__ == "__main__":
raise SystemExit(main())