forked from mtusi2026/mtusi2026lab3
Замеры образа коммита 3f45eb1 (до улучшения): фиксированный суммарный CPU
API (1x2.0 / 2x1.0 / 4x0.5), рост числа реплик против узкой БД (0.5 CPU),
рост CPU БД при 3 репликах. raw JSON на каждую ступень, results.csv,
лог прогона. analyze.py: медиана и разброс по повторам, потолок и
SLO-ёмкость, узкое место по CPU относительно фактического лимита,
SVG-графики.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
215 lines
13 KiB
Python
215 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""Track 1 analysis: evidence/track1/<series>/<label>/results.csv -> summary.md + SVG charts.
|
||
|
||
python bench/track1/analyze.py # all series found under evidence/track1
|
||
|
||
Per configuration and offered rate: median over repeats (and min..max) of SUCCESSFUL req/s
|
||
(only 201 responses count; errors and dropped iterations are failed demand), p95 of successful
|
||
requests, CPU of every component as a share of ITS OWN limit. Capacity of a configuration =
|
||
per repeat the highest successful req/s over the sweep, then median and range over repeats.
|
||
SLO capacity = the highest offered rate that was served: ok >= 97 % of offered, p95 <= 500 ms.
|
||
Bottleneck = the component whose CPU is >= 90 % of its limit at the capacity step.
|
||
Standard library only; charts are plain SVG (one y-axis, <= 4 series, legend + labels, hover titles).
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import csv
|
||
import re
|
||
import statistics
|
||
from collections import defaultdict
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
EVIDENCE = ROOT / "evidence" / "track1"
|
||
SERIES_COLORS = [("#2a78d6", "#3987e5"), ("#eb6834", "#d95926"), ("#1baf7a", "#199e70"), ("#eda100", "#c98500")]
|
||
|
||
|
||
def num(value):
|
||
if value in (None, "", "None"):
|
||
return None
|
||
return float(value)
|
||
|
||
|
||
CONFIG_RE = re.compile(r"n(?P<n>\d+)-api(?P<api>[\d.]+)-pg(?P<pg>[\d.]+)-pool(?P<pool>\d+)(?:-message_write_path_(?P<path>\w+))?")
|
||
|
||
|
||
def label_of(config: str) -> str:
|
||
m = CONFIG_RE.fullmatch(config)
|
||
if not m:
|
||
return config
|
||
text = f"{m['n']} × {m['api']} CPU API, БД {m['pg']} CPU"
|
||
if m["path"]:
|
||
text += f", запись {m['path']}"
|
||
return text
|
||
|
||
|
||
def load(path: Path) -> list[dict]:
|
||
with path.open(encoding="utf-8") as f:
|
||
return list(csv.DictReader(f))
|
||
|
||
|
||
def aggregate(rows: list[dict]) -> dict:
|
||
by_cfg: dict[str, dict] = defaultdict(lambda: defaultdict(list))
|
||
for r in rows:
|
||
by_cfg[r["config"]][int(r["rate"])].append(r)
|
||
result = {}
|
||
for cfg, by_rate in by_cfg.items():
|
||
points = []
|
||
for rate in sorted(by_rate):
|
||
rs = by_rate[rate]
|
||
ok = [num(r["ok_rps"]) for r in rs]
|
||
p95 = [num(r["p95_ms"]) for r in rs if num(r["p95_ms"]) is not None]
|
||
points.append({"rate": rate, "ok": statistics.median(ok), "lo": min(ok), "hi": max(ok), "n": len(rs),
|
||
"p95": statistics.median(p95) if p95 else None,
|
||
"err": statistics.median(num(r["error_ratio"]) for r in rs),
|
||
"dropped": statistics.median(num(r["dropped"]) for r in rs),
|
||
"api_cpu": statistics.median(num(r["api_cpu_pct_total"]) or 0 for r in rs),
|
||
"pg_cpu": statistics.median(num(r["pg_cpu_pct"]) or 0 for r in rs),
|
||
"lb_cpu": statistics.median(num(r["lb_cpu_pct"]) or 0 for r in rs),
|
||
"k6_cpu": statistics.median(num(r["k6_cpu_pct"]) or 0 for r in rs),
|
||
"pool_wait": statistics.median(num(r["pool_wait_ms_avg"]) or 0 for r in rs),
|
||
"stmts": statistics.median(num(r["statements_per_ok"]) or 0 for r in rs),
|
||
"commits": statistics.median(num(r["pg_commits_per_ok"]) or 0 for r in rs),
|
||
"chkdown": sum(int(num(r["lb_chkdown"]) or 0) for r in rs),
|
||
"api_limit": num(rs[0].get("api_cpus_limit_total")), "pg_limit": num(rs[0].get("pg_cpus_limit")),
|
||
"replicas": num(rs[0].get("replicas_running"))})
|
||
repeats = sorted({int(r["repeat"]) for rr in by_rate.values() for r in rr})
|
||
caps, slo = [], []
|
||
for rep in repeats:
|
||
rr = [r for rate_rows in by_rate.values() for r in rate_rows if int(r["repeat"]) == rep]
|
||
caps.append(max(num(r["ok_rps"]) for r in rr))
|
||
served = [int(r["rate"]) for r in rr if num(r["ok_rps"]) >= 0.97 * int(r["rate"])
|
||
and (num(r["p95_ms"]) or 1e9) <= 500]
|
||
slo.append(max(served) if served else 0)
|
||
best = max(points, key=lambda p: p["ok"])
|
||
bottleneck = []
|
||
if best["api_limit"] and best["api_cpu"] >= 90 * best["api_limit"]:
|
||
bottleneck.append("API CPU")
|
||
elif best["replicas"] and best["api_cpu"] >= 90 * best["replicas"] and best["api_limit"] > best["replicas"]:
|
||
# One uvicorn process per replica runs Python bytecode on ONE core (GIL), whatever the quota.
|
||
bottleneck.append("API: 1 ядро на процесс (GIL), квота не используется")
|
||
if best["pg_limit"] and best["pg_cpu"] >= 90 * best["pg_limit"]:
|
||
bottleneck.append("PostgreSQL CPU")
|
||
if best["lb_cpu"] >= 90:
|
||
bottleneck.append("LB CPU")
|
||
result[cfg] = {"points": points, "capacity": statistics.median(caps), "cap_lo": min(caps), "cap_hi": max(caps),
|
||
"slo": statistics.median(slo), "repeats": len(repeats), "best": best,
|
||
"bottleneck": ", ".join(bottleneck) or "нет (ни один CPU не у предела)"}
|
||
return result
|
||
|
||
|
||
def svg_chart(title: str, series: dict[str, list[dict]], path: Path, y_key: str = "ok",
|
||
y_label: str = "успешных запросов/с (201)", diagonal: bool = True) -> None:
|
||
width, height, left, right, top, bottom = 760, 420, 64, 210, 40, 52
|
||
xs = [p["rate"] for pts in series.values() for p in pts]
|
||
ys = [p.get("hi", p[y_key]) for pts in series.values() for p in pts] + ([max(xs)] if diagonal else [])
|
||
x0, x1 = 0, max(xs) * 1.05
|
||
y1 = max(ys) * 1.1
|
||
sx = lambda v: left + (v - x0) / (x1 - x0) * (width - left - right) # noqa: E731
|
||
sy = lambda v: height - bottom - v / y1 * (height - top - bottom) # noqa: E731
|
||
out = [f'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 {width} {height}" font-family="system-ui, sans-serif" font-size="12">',
|
||
"<style>.bg{fill:#fcfcfb}.ink{fill:#0b0b0b}.ink2{fill:#52514e}.grid{stroke:#e4e3df}.ref{stroke:#b5b3ac}"
|
||
+ "".join(f".s{i}{{stroke:{c[0]};fill:{c[0]}}}" for i, c in enumerate(SERIES_COLORS))
|
||
+ "@media (prefers-color-scheme: dark){.bg{fill:#1a1a19}.ink{fill:#fff}.ink2{fill:#c3c2b7}.grid{stroke:#34332f}.ref{stroke:#6b6a64}"
|
||
+ "".join(f".s{i}{{stroke:{c[1]};fill:{c[1]}}}" for i, c in enumerate(SERIES_COLORS)) + "}</style>",
|
||
f'<rect class="bg" width="{width}" height="{height}"/>',
|
||
f'<text class="ink" x="{left}" y="22" font-size="14" font-weight="600">{title}</text>']
|
||
step = nice_step(y1)
|
||
v = 0.0
|
||
while v <= y1:
|
||
out.append(f'<line class="grid" x1="{left}" x2="{width - right}" y1="{sy(v):.1f}" y2="{sy(v):.1f}" stroke-width="1"/>')
|
||
out.append(f'<text class="ink2" x="{left - 8}" y="{sy(v) + 4:.1f}" text-anchor="end">{v:g}</text>')
|
||
v += step
|
||
for rate in sorted(set(xs)):
|
||
out.append(f'<text class="ink2" x="{sx(rate):.1f}" y="{height - bottom + 18}" text-anchor="middle">{rate}</text>')
|
||
out.append(f'<text class="ink2" x="{(left + width - right) / 2}" y="{height - 12}" text-anchor="middle">предложенная нагрузка, запросов/с (open-loop)</text>')
|
||
out.append(f'<text class="ink2" transform="translate(16 {(top + height - bottom) / 2}) rotate(-90)" text-anchor="middle">{y_label}</text>')
|
||
if diagonal:
|
||
out.append(f'<line class="ref" x1="{sx(0):.1f}" y1="{sy(0):.1f}" x2="{sx(max(xs)):.1f}" y2="{sy(max(xs)):.1f}" stroke-width="1"/>')
|
||
out.append(f'<text class="ink2" x="{sx(max(xs)) - 4:.1f}" y="{sy(max(xs)) - 6:.1f}" text-anchor="end">обслужено = предложено</text>')
|
||
legend_y = top + 10
|
||
for i, (name, pts) in enumerate(series.items()):
|
||
cls = f"s{i}"
|
||
line = " ".join(f"{sx(p['rate']):.1f},{sy(p[y_key]):.1f}" for p in pts)
|
||
out.append(f'<polyline class="{cls}" points="{line}" fill="none" stroke-width="2" stroke-linejoin="round" stroke-linecap="round"/>')
|
||
for p in pts:
|
||
if "lo" in p and y_key == "ok":
|
||
out.append(f'<line class="{cls}" x1="{sx(p["rate"]):.1f}" x2="{sx(p["rate"]):.1f}" y1="{sy(p["lo"]):.1f}" y2="{sy(p["hi"]):.1f}" stroke-width="2" opacity="0.5"/>')
|
||
tip = (f"{name}: предложено {p['rate']}/с, успешно {p[y_key]:g}/с" + (f" (повторы {p['lo']:g}–{p['hi']:g})" if "lo" in p else "")
|
||
+ (f", p95 {p['p95']:.0f} мс" if p.get("p95") else ""))
|
||
out.append(f'<circle class="{cls}" cx="{sx(p["rate"]):.1f}" cy="{sy(p[y_key]):.1f}" r="4" stroke="#fcfcfb" stroke-width="2"><title>{tip}</title></circle>')
|
||
last = pts[-1]
|
||
out.append(f'<text class="ink2" x="{sx(last["rate"]) + 8:.1f}" y="{sy(last[y_key]) + 4:.1f}">{last[y_key]:.0f}</text>')
|
||
out.append(f'<line class="{cls}" x1="{width - right + 34}" x2="{width - right + 52}" y1="{legend_y}" y2="{legend_y}" stroke-width="2"/>')
|
||
for k, chunk in enumerate(wrap(name, 22)):
|
||
out.append(f'<text class="ink" x="{width - right + 58}" y="{legend_y + 4 + 14 * k}">{chunk}</text>')
|
||
legend_y += 14 * len(wrap(name, 22)) + 12
|
||
out.append("</svg>")
|
||
path.write_text("\n".join(out) + "\n", encoding="utf-8")
|
||
|
||
|
||
def wrap(text: str, width: int) -> list[str]:
|
||
words, lines, cur = text.split(), [], ""
|
||
for w in words:
|
||
if len(cur) + len(w) + 1 > width and cur:
|
||
lines.append(cur)
|
||
cur = w
|
||
else:
|
||
cur = f"{cur} {w}".strip()
|
||
return lines + [cur]
|
||
|
||
|
||
def nice_step(top: float) -> float:
|
||
for step in (10, 20, 25, 50, 100, 200, 250, 500, 1000):
|
||
if top / step <= 8:
|
||
return step
|
||
return 2000
|
||
|
||
|
||
def series_section(series_dir: Path) -> list[str]:
|
||
lines = []
|
||
for label_dir in sorted(p for p in series_dir.iterdir() if (p / "results.csv").exists()):
|
||
rows = load(label_dir / "results.csv")
|
||
if not rows:
|
||
continue # series still running / not started
|
||
agg = aggregate(rows)
|
||
name = f"{series_dir.name}/{label_dir.name}"
|
||
lines += [f"### {name}", "", "| Конфигурация | Повторов | Потолок, успешных/с (медиана, мин–макс) | SLO-ёмкость, предложено/с | "
|
||
"p95 на потолке, мс | CPU API / лимит | CPU БД / лимит | CPU LB | CPU k6 | Ожидание пула, мс | SQL-операторов на сообщение | Узкое место |",
|
||
"| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"]
|
||
for cfg, a in agg.items():
|
||
b = a["best"]
|
||
lines.append(f"| {label_of(cfg)} | {a['repeats']} | **{a['capacity']:.0f}** ({a['cap_lo']:.0f}–{a['cap_hi']:.0f}) | {a['slo']:.0f} | "
|
||
f"{b['p95']:.0f} | {b['api_cpu']:.0f} % / {100 * (b['api_limit'] or 0):.0f} % | "
|
||
f"{b['pg_cpu']:.0f} % / {100 * (b['pg_limit'] or 0):.0f} % | {b['lb_cpu']:.0f} % | {b['k6_cpu']:.0f} % | "
|
||
f"{b['pool_wait']:.1f} | {b['stmts']:.1f} | {a['bottleneck']} |")
|
||
lines += ["", "<details><summary>Все ступени (медианы по повторам)</summary>", "",
|
||
"| Конфигурация | Предложено/с | Успешно/с (мин–макс) | p95, мс | Ошибки | Dropped | CPU API | CPU БД | Ожидание пула, мс | DOWN в LB |",
|
||
"| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"]
|
||
for cfg, a in agg.items():
|
||
for p in a["points"]:
|
||
lines.append(f"| {label_of(cfg)} | {p['rate']} | {p['ok']:.1f} ({p['lo']:.0f}–{p['hi']:.0f}) | "
|
||
f"{(p['p95'] or 0):.0f} | {p['err']:.4f} | {p['dropped']:.0f} | {p['api_cpu']:.0f} % | {p['pg_cpu']:.0f} % | "
|
||
f"{p['pool_wait']:.1f} | {p['chkdown']} |")
|
||
lines += ["", "</details>", ""]
|
||
chart = label_dir / "throughput.svg"
|
||
svg_chart(name, {label_of(c): a["points"] for c, a in agg.items()}, chart)
|
||
lines += [f".as_posix()})", ""]
|
||
return lines
|
||
|
||
|
||
def main() -> int:
|
||
lines = ["# Трек 1 — сводка (сгенерировано bench/track1/analyze.py из results.csv)", "",
|
||
"Успешные = только ответы 201 в фазе measure; ошибки и dropped iterations k6 — неудовлетворённый спрос. "
|
||
"CPU в % одного ядра (100 % = 1 CPU), медиана выборок docker stats за фазу measure; у API — сумма реплик в каждый "
|
||
"момент. Лимиты — фактические из docker inspect.", ""]
|
||
for series_dir in sorted(p for p in EVIDENCE.iterdir() if p.is_dir() and p.name[:1].isupper()):
|
||
lines += [f"## {series_dir.name}", ""] + series_section(series_dir)
|
||
(EVIDENCE / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
print((EVIDENCE / "summary.md").read_text(encoding="utf-8")[:4000])
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|