#!/usr/bin/env python3 """Aggregate track 2 raw data into tables (summary.md) and charts (plans.png). python bench/track2/summarize.py # matplotlib from requirements-bench.txt for the chart """ from __future__ import annotations import csv import json import statistics import sys from collections import defaultdict from pathlib import Path ROOT = Path(__file__).resolve().parents[2] OUT = ROOT / "evidence/track2" SIZE_ORDER = {"big": 0, "medium": 1, "small": 2} def load(name: str) -> list[dict]: with (OUT / name).open(encoding="utf-8") as f: return list(csv.DictReader(f)) def agg(rows: list[dict], keys: tuple[str, ...], fields: tuple[str, ...]) -> dict: groups: dict = defaultdict(list) for r in rows: groups[tuple(r[k] for k in keys)].append(r) out = {} for k, g in groups.items(): out[k] = {f: statistics.median(float(r[f]) for r in g) for f in fields} out[k]["tps_min"] = min(float(r["tps"]) for r in g) out[k]["tps_max"] = max(float(r["tps"]) for r in g) out[k]["n"] = len(g) return out def fmt(x: float) -> str: return f"{x:,.0f}".replace(",", " ") if x >= 100 else (f"{x:.1f}" if x >= 10 else f"{x:.2f}") def main() -> int: sys.stdout.reconfigure(encoding="utf-8") reads, writes, wal = load("reads.csv"), load("writes.csv"), load("wal.csv") sizes_rows = {r["variant"]: r for r in load("index-sizes.csv")} meta = json.loads((OUT / "plans-meta.json").read_text(encoding="utf-8")) r = agg(reads, ("variant", "query", "chat"), ("tps", "p50_ms", "p95_ms", "p99_ms")) w = agg(writes, ("variant",), ("tps", "p50_ms", "p95_ms", "p99_ms")) wal_by = {v: [int(r["wal_bytes_per_insert"]) for r in wal if r["variant"] == v] for v in {r["variant"] for r in wal}} for v, values in wal_by.items(): w[(v,)]["wal_bytes_per_insert"] = statistics.median(values) w[(v,)]["wal_min"], w[(v,)]["wal_max"] = min(values), max(values) sizes = {"big": 60000, "medium": 15000, "small": 80} md = ["## Чтение: одна страница (LIMIT 51) со случайной позиции чата", "", "Медиана 3 повторов по 10 с, 4 клиента pgbench, prepared statements. TPS min–max — разброс повторов.", "", "| Индекс | Запрос | Чат (сообщений) | TPS (min–max) | p50, мс | p95, мс | p99, мс |", "| --- | --- | ---: | ---: | ---: | ---: | ---: |"] for (variant, query, chat), v in sorted(r.items(), key=lambda kv: (kv[0][0] != "keyset_index", kv[0][0], kv[0][1], SIZE_ORDER[kv[0][2]])): md.append(f"| {variant} | {query} | {chat} ({sizes[chat]:,}) | {fmt(v['tps'])} ({fmt(v['tps_min'])}–{fmt(v['tps_max'])}) " f"| {fmt(v['p50_ms'])} | {fmt(v['p95_ms'])} | {fmt(v['p99_ms'])} |".replace(",", " ", 1)) md += ["", "## Запись: транзакция сообщения приложения (UPDATE chats … RETURNING + INSERT)", "", "TPS и задержка — медиана 5 повторов по 10 с, 8 клиентов. WAL — медиана 3 детерминированных прогонов: " "5 000 транзакций, 1 клиент, autovacuum выключен, CHECKPOINT перед прогоном.", "", "| Индексы messages | TPS (min–max) | p50, мс | p95, мс | p99, мс | WAL, байт/вставку (min–max) | Доп. индексы на 121 120 строк, МиБ |", "| --- | ---: | ---: | ---: | ---: | ---: | ---: |"] for variant in ("no_index", "keyset_index", "keyset_plus_sender"): v = w[(variant,)] idx_mib = int(sizes_rows[variant]["secondary_index_bytes"]) / 2**20 # same dataset for every variant md.append(f"| {variant} | {fmt(v['tps'])} ({fmt(v['tps_min'])}–{fmt(v['tps_max'])}) | {fmt(v['p50_ms'])} | {fmt(v['p95_ms'])} " f"| {fmt(v['p99_ms'])} | {v['wal_bytes_per_insert']:.0f} ({v['wal_min']}–{v['wal_max']}) | {idx_mib:.1f} |") md += ["", "Размеры индексов измерены отдельно на одинаковых 121 120 строках (bench/track2/index_sizes.py); " "первичный ключ во всех вариантах — " + f"{int(sizes_rows['no_index']['pkey_bytes']) / 2**20:.1f} МиБ, таблица — " f"{int(sizes_rows['no_index']['table_bytes']) / 2**20:.1f} МиБ."] (OUT / "summary.md").write_text("\n".join(md) + "\n", encoding="utf-8", newline="\n") print("\n".join(md)) try: import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt except ImportError: print("matplotlib is not installed: skipping plans.png") return 0 fig, axes = plt.subplots(1, 2, figsize=(15, 5)) chats = ["small", "medium", "big"] x = [sizes[c] for c in chats] styles = {("keyset_index", "keyset"): ("#12a150", "-o"), ("keyset_index", "offset"): ("#12a150", "--s"), ("chat_id_index", "keyset"): ("#e07b00", "-o"), ("chat_id_index", "offset"): ("#e07b00", "--s"), ("no_index", "keyset"): ("#b03060", "-o"), ("no_index", "offset"): ("#b03060", "--s")} for (variant, query), (color, style) in styles.items(): axes[0].plot(x, [r[(variant, query, c)]["p95_ms"] for c in chats], style, color=color, label=f"{variant}, {query}") axes[0].set_xscale("log") axes[0].set_yscale("log") axes[0].set_xlabel("messages in the chat") axes[0].set_ylabel("p95 latency of one page, ms (log)") axes[0].set_title("Read: random page of a chat") axes[0].grid(True, which="both", alpha=0.3) axes[0].legend(fontsize=8) names = ["no_index", "keyset_index", "keyset_plus_sender"] axes[1].bar([n.replace("_", "\n") for n in names], [w[(n,)]["wal_bytes_per_insert"] for n in names], color="#2a6fdb") for i, n in enumerate(names): axes[1].text(i, w[(n,)]["wal_bytes_per_insert"], f"{w[(n,)]['tps']:.0f} TPS", ha="center", va="bottom") axes[1].set_ylabel("WAL bytes per inserted message") axes[1].set_title("Write cost of indexes (WAL: 5000 inserts; TPS: 8 clients)") fig.suptitle("Track 2: 121 120 messages, chats of 60 000 / 15 000 / 2 000 / 80; PostgreSQL 18.6, 2 CPU / 2 GiB") fig.tight_layout() fig.savefig(OUT / "plans.png", dpi=120) print("chart:", OUT / "plans.png") return 0 if __name__ == "__main__": sys.exit(main())