forked from mtusi2026/mtusi2026lab2
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
114 lines
6.3 KiB
Python
114 lines
6.3 KiB
Python
#!/usr/bin/env python3
|
||
"""Aggregate track 2 raw data into tables (summary.md) and charts (plans.png).
|
||
|
||
python bench/track2/summarize.py # matplotlib from requirements-bench.txt for the chart
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import csv
|
||
import json
|
||
import statistics
|
||
import sys
|
||
from collections import defaultdict
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
OUT = ROOT / "evidence/track2"
|
||
SIZE_ORDER = {"big": 0, "medium": 1, "small": 2}
|
||
|
||
|
||
def load(name: str) -> list[dict]:
|
||
with (OUT / name).open(encoding="utf-8") as f:
|
||
return list(csv.DictReader(f))
|
||
|
||
|
||
def agg(rows: list[dict], keys: tuple[str, ...], fields: tuple[str, ...]) -> dict:
|
||
groups: dict = defaultdict(list)
|
||
for r in rows:
|
||
groups[tuple(r[k] for k in keys)].append(r)
|
||
out = {}
|
||
for k, g in groups.items():
|
||
out[k] = {f: statistics.median(float(r[f]) for r in g) for f in fields}
|
||
out[k]["tps_min"] = min(float(r["tps"]) for r in g)
|
||
out[k]["tps_max"] = max(float(r["tps"]) for r in g)
|
||
out[k]["n"] = len(g)
|
||
return out
|
||
|
||
|
||
def fmt(x: float) -> str:
|
||
return f"{x:,.0f}".replace(",", " ") if x >= 100 else (f"{x:.1f}" if x >= 10 else f"{x:.2f}")
|
||
|
||
|
||
def main() -> int:
|
||
sys.stdout.reconfigure(encoding="utf-8")
|
||
reads, writes, wal = load("reads.csv"), load("writes.csv"), load("wal.csv")
|
||
sizes_rows = {r["variant"]: r for r in load("index-sizes.csv")}
|
||
meta = json.loads((OUT / "plans-meta.json").read_text(encoding="utf-8"))
|
||
r = agg(reads, ("variant", "query", "chat"), ("tps", "p50_ms", "p95_ms", "p99_ms"))
|
||
w = agg(writes, ("variant",), ("tps", "p50_ms", "p95_ms", "p99_ms"))
|
||
wal_by = {v: [int(r["wal_bytes_per_insert"]) for r in wal if r["variant"] == v] for v in {r["variant"] for r in wal}}
|
||
for v, values in wal_by.items():
|
||
w[(v,)]["wal_bytes_per_insert"] = statistics.median(values)
|
||
w[(v,)]["wal_min"], w[(v,)]["wal_max"] = min(values), max(values)
|
||
sizes = {"big": 60000, "medium": 15000, "small": 80}
|
||
md = ["## Чтение: одна страница (LIMIT 51) со случайной позиции чата", "",
|
||
"Медиана 3 повторов по 10 с, 4 клиента pgbench, prepared statements. TPS min–max — разброс повторов.", "",
|
||
"| Индекс | Запрос | Чат (сообщений) | TPS (min–max) | p50, мс | p95, мс | p99, мс |",
|
||
"| --- | --- | ---: | ---: | ---: | ---: | ---: |"]
|
||
for (variant, query, chat), v in sorted(r.items(), key=lambda kv: (kv[0][0] != "keyset_index", kv[0][0], kv[0][1], SIZE_ORDER[kv[0][2]])):
|
||
md.append(f"| {variant} | {query} | {chat} ({sizes[chat]:,}) | {fmt(v['tps'])} ({fmt(v['tps_min'])}–{fmt(v['tps_max'])}) "
|
||
f"| {fmt(v['p50_ms'])} | {fmt(v['p95_ms'])} | {fmt(v['p99_ms'])} |".replace(",", " ", 1))
|
||
md += ["", "## Запись: транзакция сообщения приложения (UPDATE chats … RETURNING + INSERT)", "",
|
||
"TPS и задержка — медиана 5 повторов по 10 с, 8 клиентов. WAL — медиана 3 детерминированных прогонов: "
|
||
"5 000 транзакций, 1 клиент, autovacuum выключен, CHECKPOINT перед прогоном.", "",
|
||
"| Индексы messages | TPS (min–max) | p50, мс | p95, мс | p99, мс | WAL, байт/вставку (min–max) | Доп. индексы на 121 120 строк, МиБ |",
|
||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |"]
|
||
for variant in ("no_index", "keyset_index", "keyset_plus_sender"):
|
||
v = w[(variant,)]
|
||
idx_mib = int(sizes_rows[variant]["secondary_index_bytes"]) / 2**20 # same dataset for every variant
|
||
md.append(f"| {variant} | {fmt(v['tps'])} ({fmt(v['tps_min'])}–{fmt(v['tps_max'])}) | {fmt(v['p50_ms'])} | {fmt(v['p95_ms'])} "
|
||
f"| {fmt(v['p99_ms'])} | {v['wal_bytes_per_insert']:.0f} ({v['wal_min']}–{v['wal_max']}) | {idx_mib:.1f} |")
|
||
md += ["", "Размеры индексов измерены отдельно на одинаковых 121 120 строках (bench/track2/index_sizes.py); "
|
||
"первичный ключ во всех вариантах — " + f"{int(sizes_rows['no_index']['pkey_bytes']) / 2**20:.1f} МиБ, таблица — "
|
||
f"{int(sizes_rows['no_index']['table_bytes']) / 2**20:.1f} МиБ."]
|
||
(OUT / "summary.md").write_text("\n".join(md) + "\n", encoding="utf-8", newline="\n")
|
||
print("\n".join(md))
|
||
|
||
try:
|
||
import matplotlib
|
||
matplotlib.use("Agg")
|
||
import matplotlib.pyplot as plt
|
||
except ImportError:
|
||
print("matplotlib is not installed: skipping plans.png")
|
||
return 0
|
||
fig, axes = plt.subplots(1, 2, figsize=(15, 5))
|
||
chats = ["small", "medium", "big"]
|
||
x = [sizes[c] for c in chats]
|
||
styles = {("keyset_index", "keyset"): ("#12a150", "-o"), ("keyset_index", "offset"): ("#12a150", "--s"),
|
||
("chat_id_index", "keyset"): ("#e07b00", "-o"), ("chat_id_index", "offset"): ("#e07b00", "--s"),
|
||
("no_index", "keyset"): ("#b03060", "-o"), ("no_index", "offset"): ("#b03060", "--s")}
|
||
for (variant, query), (color, style) in styles.items():
|
||
axes[0].plot(x, [r[(variant, query, c)]["p95_ms"] for c in chats], style, color=color, label=f"{variant}, {query}")
|
||
axes[0].set_xscale("log")
|
||
axes[0].set_yscale("log")
|
||
axes[0].set_xlabel("messages in the chat")
|
||
axes[0].set_ylabel("p95 latency of one page, ms (log)")
|
||
axes[0].set_title("Read: random page of a chat")
|
||
axes[0].grid(True, which="both", alpha=0.3)
|
||
axes[0].legend(fontsize=8)
|
||
names = ["no_index", "keyset_index", "keyset_plus_sender"]
|
||
axes[1].bar([n.replace("_", "\n") for n in names], [w[(n,)]["wal_bytes_per_insert"] for n in names], color="#2a6fdb")
|
||
for i, n in enumerate(names):
|
||
axes[1].text(i, w[(n,)]["wal_bytes_per_insert"], f"{w[(n,)]['tps']:.0f} TPS", ha="center", va="bottom")
|
||
axes[1].set_ylabel("WAL bytes per inserted message")
|
||
axes[1].set_title("Write cost of indexes (WAL: 5000 inserts; TPS: 8 clients)")
|
||
fig.suptitle("Track 2: 121 120 messages, chats of 60 000 / 15 000 / 2 000 / 80; PostgreSQL 18.6, 2 CPU / 2 GiB")
|
||
fig.tight_layout()
|
||
fig.savefig(OUT / "plans.png", dpi=120)
|
||
print("chart:", OUT / "plans.png")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|