Files
mtusi2026lab2/bench/track2/plans.py
T
Alexey BodanyukandClaude Opus 5.5 eb92e8e91e bench: трек 2 — планы запросов, стоимость записи, backup/restore со сверкой
plans.py: 121 120 сообщений (60k/15k/2k/80), 3 индекса x keyset/OFFSET x 3 размера
чата, pgbench в отдельном контейнере; WAL на вставку детерминированно.
backup_restore.py: pg_dump живой БД во время записи через API, восстановление в
новый контейнер и volume, сверка MD5 по чатам, API на восстановленной БД.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 20:08:13 +03:00

287 lines
15 KiB
Python

#!/usr/bin/env python3
"""Track 2, part 1: query plans and write cost on >= 100 000 messages with uneven chat sizes.
python bench/track2/plans.py # ~20 min, results in evidence/track2/
An ISOLATED PostgreSQL container (same pinned image as compose.yaml, --cpus 2 --memory 2g,
no published ports) gets the schema from migrations/*.sql and a deterministic synthetic
dataset: 1 chat x 60 000, 2 x 15 000, 8 x 2 000, 189 x 80 messages = 121 120 messages,
texts of 10..400 bytes. The live stand is not touched: indexes are dropped and created
here freely.
For every index variant the script runs pgbench (prepared statements, like the driver):
* reads — one history page (LIMIT 51) after a RANDOM position of a big/medium/small chat,
keyset query (the app's SQL) vs OFFSET; 4 clients;
* writes — the app's message transaction (UPDATE chats ... RETURNING + INSERT); 8 clients;
and records TPS, p50/p95/p99 latency, WAL bytes per insert, index/table sizes, EXPLAIN.
"""
from __future__ import annotations
import argparse
import csv
import json
import math
import platform
import secrets
import statistics
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "src"))
from messenger.repo import HISTORY_AFTER # noqa: E402 (the app's real statement)
IMAGE = "postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c"
NAME = "mtusi-track2-pg"
CLIENT = "mtusi-track2-client" # pgbench runs here: it must not eat the server's CPU quota
PASSWORD = secrets.token_urlsafe(16) # throwaway, never printed
USER_ID = "00000000-0000-4000-8000-0000000b0001"
VARIANTS = {
# name: (DDL of secondary indexes on messages, description)
"keyset_index": (["CREATE INDEX messages_chat_keyset ON messages (chat_id, created_at, id)"],
"(chat_id, created_at, id) — used by the app"),
"chat_id_index": (["CREATE INDEX messages_chat_only ON messages (chat_id)"],
"(chat_id) only"),
"no_index": ([], "primary key only"),
"keyset_plus_sender": (["CREATE INDEX messages_chat_keyset ON messages (chat_id, created_at, id)",
"CREATE INDEX messages_sender_time ON messages (sender_id, created_at)"],
"app index + a second index (sender_id, created_at)"),
}
READ_VARIANTS = ["keyset_index", "chat_id_index", "no_index"]
WRITE_VARIANTS = ["no_index", "keyset_index", "keyset_plus_sender"]
KINDS = {"big": 60_000, "medium": 15_000, "small": 80}
def sh(args: list[str], input_text: str | None = None, check: bool = True) -> str:
result = subprocess.run(args, input=input_text, capture_output=True, text=True, encoding="utf-8")
if check and result.returncode != 0:
raise RuntimeError(f"{' '.join(args[:4])}: {result.stderr.strip()[-1500:]}")
return result.stdout
def psql(sql: str) -> str:
return sh(["docker", "exec", "-i", NAME, "psql", "-U", "postgres", "-d", "bench", "-X", "-q", "-At",
"-v", "ON_ERROR_STOP=1", "-f", "-"], input_text=sql).strip()
def put(path: str, content: str) -> None:
sh(["docker", "exec", "-i", CLIENT, "sh", "-c", f"cat > {path}"], input_text=content)
def start_container(cpus: str, memory: str) -> None:
sh(["docker", "rm", "-f", "-v", CLIENT, NAME], check=False)
sh(["docker", "run", "-d", "--name", NAME, "--cpus", cpus, "--memory", memory,
"-e", f"POSTGRES_PASSWORD={PASSWORD}", "-e", "POSTGRES_DB=bench", IMAGE])
# Load generator: separate container (own CPU quota) sharing the server's network namespace.
sh(["docker", "run", "-d", "--name", CLIENT, "--network", f"container:{NAME}", "--cpus", "2",
"-e", f"PGPASSWORD={PASSWORD}", "--entrypoint", "sleep", IMAGE, "infinity"])
for _ in range(120):
if subprocess.run(["docker", "exec", NAME, "pg_isready", "-U", "postgres", "-d", "bench", "-q"]).returncode == 0:
time.sleep(1)
return
time.sleep(0.5)
raise RuntimeError("experiment PostgreSQL did not start")
SEED = f"""
SELECT setseed(0.42);
INSERT INTO users (id, username, display_name, password_hash)
VALUES ('{USER_ID}', 'track2_bench', convert_to('Track 2', 'UTF8'), 'not-a-login');
CREATE TABLE bench_sizes AS
SELECT n, CASE WHEN n = 1 THEN 60000 WHEN n <= 3 THEN 15000 WHEN n <= 11 THEN 2000 ELSE 80 END AS size,
md5('track2-chat-' || n)::uuid AS chat_id
FROM generate_series(1::int, 200::int) AS n;
INSERT INTO chats (id, title, created_by, created_at)
SELECT chat_id, convert_to('chat ' || n, 'UTF8'), '{USER_ID}', timestamptz '2026-01-01 00:00:00+00' FROM bench_sizes;
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
SELECT chat_id, '{USER_ID}', 0, timestamptz '2026-01-01 00:00:00+00' FROM bench_sizes;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), s.chat_id, '{USER_ID}',
convert_to(left(repeat('сообщение ', 40), 5 + (random() * 195)::int), 'UTF8'),
timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
FROM bench_sizes s CROSS JOIN LATERAL generate_series(1::int, s.size) AS g;
UPDATE chats c SET last_message_at = m.mx FROM (SELECT chat_id, max(created_at) mx FROM messages GROUP BY chat_id) m
WHERE m.chat_id = c.id;
CREATE TABLE bench_chats AS SELECT n, chat_id AS id FROM bench_sizes;
CREATE TABLE bench_cursors AS
SELECT kind, row_number() OVER (PARTITION BY kind ORDER BY rn)::int AS n, created_at AS ts, id AS mid, rn::int AS off
FROM (SELECT CASE s.n WHEN 1 THEN 'big' WHEN 2 THEN 'medium' ELSE 'small' END AS kind, m.created_at, m.id,
row_number() OVER (PARTITION BY m.chat_id ORDER BY m.created_at, m.id) AS rn, s.size
FROM messages m JOIN bench_sizes s ON s.chat_id = m.chat_id WHERE s.n IN (1, 2, 12)) x
WHERE rn % greatest(1, size / 200) = 0 AND rn < size;
CREATE INDEX ON bench_cursors (kind, n);
VACUUM ANALYZE;
"""
def pg_param(sql: str) -> str:
"""psycopg %s placeholders -> pgbench :variables (passed as parameters in prepared mode)."""
names = iter([":chat", ":cts", ":cid", "51"])
out = sql
while "%s" in out:
out = out.replace("%s", next(names), 1)
return out
READ_SCRIPTS = {
"keyset": "\\set k random(1, :ncur)\n"
"SELECT ts AS cts, mid AS cid FROM bench_cursors WHERE kind = :kind AND n = :k \\gset\n"
+ pg_param(HISTORY_AFTER) + ";\n",
"offset": "\\set k random(1, :ncur)\n"
"SELECT off FROM bench_cursors WHERE kind = :kind AND n = :k \\gset\n"
"SELECT id, sender_id, text, created_at FROM messages WHERE chat_id = :chat "
"ORDER BY created_at, id OFFSET :off LIMIT 51;\n",
}
WRITE_SCRIPT = (
"\\set c random(1, 200)\n"
"SELECT id AS chat FROM bench_chats WHERE n = :c \\gset\n"
"BEGIN;\n"
"UPDATE chats SET last_message_at = greatest(clock_timestamp(), last_message_at + interval '1 microsecond') "
"WHERE id = :chat RETURNING last_message_at AS ts \\gset\n"
f"INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES "
f"(gen_random_uuid(), :chat, '{USER_ID}', convert_to(repeat('w', 120), 'UTF8'), :ts);\n"
"END;\n"
)
def pgbench(script_path: str, clients: int, seconds: int, defines: dict[str, str]) -> dict:
"""Run pgbench with per-transaction logging; return TPS and latency percentiles (ms)."""
sh(["docker", "exec", CLIENT, "sh", "-c", "rm -f /tmp/pgblog*"])
args = ["docker", "exec", CLIENT, "sh", "-c",
"cd /tmp && pgbench -h 127.0.0.1 -U postgres -n -M prepared -c {c} -j {c} -T {t} -l --log-prefix=pgblog {d} -f {f} bench".format(
c=clients, t=seconds, f=script_path, d=" ".join(f"-D {k}={v}" for k, v in defines.items()))]
out = sh(args)
tps = float(next(l for l in out.splitlines() if l.startswith("tps = ")).split()[2])
failed = next((l for l in out.splitlines() if "number of failed transactions" in l), "0").split(":")[-1].split()[0]
lat = sorted(int(line.split()[2]) / 1000 for line in sh(["docker", "exec", CLIENT, "sh", "-c", "cat /tmp/pgblog*"]).splitlines())
def pct(p: float) -> float:
return round(lat[max(0, math.ceil(p * len(lat)) - 1)], 3)
return {"tps": round(tps, 1), "transactions": len(lat), "failed": int(failed),
"p50_ms": pct(.50), "p95_ms": pct(.95), "p99_ms": pct(.99), "mean_ms": round(statistics.mean(lat), 3)}
def use_variant(name: str) -> dict:
psql("DROP INDEX IF EXISTS messages_chat_keyset; DROP INDEX IF EXISTS messages_chat_only; "
"DROP INDEX IF EXISTS messages_sender_time;")
started = time.monotonic()
for ddl in VARIANTS[name][0]:
psql(ddl + ";")
build_s = round(time.monotonic() - started, 2)
psql("VACUUM ANALYZE messages;")
sizes = psql("SELECT indexrelname || '=' || pg_relation_size(indexrelid) FROM pg_stat_user_indexes "
"WHERE relname = 'messages' ORDER BY 1;").split()
return {"index_build_s": build_s, "index_bytes": dict(s.split("=") for s in sizes),
"table_bytes": int(psql("SELECT pg_relation_size('messages');"))}
def explain_late(kind: str) -> str:
chat = psql(f"SELECT chat_id FROM bench_sizes WHERE n = {dict(big=1, medium=2, small=12)[kind]};")
return psql(f"""
SET plan_cache_mode = force_generic_plan;
PREPARE p(uuid, timestamptz, uuid, int) AS {HISTORY_AFTER.replace('%s', '$1', 1).replace('%s', '$2', 1).replace('%s', '$3', 1).replace('%s', '$4', 1)};
SELECT ts, mid FROM bench_cursors WHERE kind = '{kind}' ORDER BY n DESC LIMIT 1 \\gset
EXPLAIN (ANALYZE, BUFFERS) EXECUTE p('{chat}', :'ts', :'mid', 51);
""")
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--seconds", type=int, default=10)
parser.add_argument("--warmup", type=int, default=3)
parser.add_argument("--repeats", type=int, default=3)
parser.add_argument("--out", default=str(ROOT / "evidence/track2"))
parser.add_argument("--writes-only", action="store_true", help="re-measure writes, keep reads.csv")
parser.add_argument("--write-repeats", type=int, default=5)
args = parser.parse_args()
out = Path(args.out)
(out / "explain").mkdir(parents=True, exist_ok=True)
start_container("2", "2g")
schema = "".join(p.read_text(encoding="utf-8") for p in sorted((ROOT / "migrations").glob("*.sql")))
psql(schema)
seed_started = time.monotonic()
psql(SEED)
seed_s = round(time.monotonic() - seed_started, 1)
stats = psql("SELECT count(*), count(DISTINCT chat_id), min(octet_length(text)), max(octet_length(text)) FROM messages;")
sizes = psql("SELECT size || 'x' || count(*) FROM bench_sizes GROUP BY size ORDER BY size DESC;").split()
for name, script in READ_SCRIPTS.items():
put(f"/tmp/read_{name}.sql", script)
put("/tmp/write.sql", WRITE_SCRIPT)
chats = {k: psql(f"SELECT chat_id FROM bench_sizes WHERE n = {dict(big=1, medium=2, small=12)[k]};") for k in KINDS}
ncur = {k: int(psql(f"SELECT count(*) FROM bench_cursors WHERE kind = '{k}';")) for k in KINDS}
meta = {"image": IMAGE, "limits": {"server_cpus": 2, "server_memory": "2g", "pgbench_client_cpus": 2}, "pg_settings": "image defaults",
"dataset": {"messages_chats_minbytes_maxbytes": stats, "chat_sizes": sizes, "seed_seconds": seed_s},
"pgbench": {"mode": "prepared", "seconds": args.seconds, "warmup": args.warmup, "repeats": args.repeats,
"read_clients": 4, "write_clients": 8, "page": "LIMIT 51 after a random position"},
"host": {"platform": platform.platform(), "docker": sh(["docker", "version", "-f", "{{.Server.Version}}"]).strip()},
"variants": {k: v[1] for k, v in VARIANTS.items()}, "sizes": {}}
print("dataset:", stats, sizes, flush=True)
reads, writes = [], []
if args.writes_only:
old = json.loads((out / "plans-meta.json").read_text(encoding="utf-8"))
meta["sizes"] = old.get("sizes", {})
for variant in ([] if args.writes_only else READ_VARIANTS):
meta["sizes"][variant] = use_variant(variant)
for kind in KINDS:
(out / "explain" / f"{variant}-{kind}-late-page.txt").write_text(explain_late(kind), encoding="utf-8", newline="\n")
for rep in range(1, args.repeats + 1):
for query in READ_SCRIPTS:
for kind in KINDS:
defines = {"chat": chats[kind], "kind": kind, "ncur": str(ncur[kind])}
pgbench(f"/tmp/read_{query}.sql", 4, args.warmup, defines) # warm-up, discarded
row = {"variant": variant, "query": query, "chat": kind, "chat_size": KINDS[kind], "repeat": rep,
**pgbench(f"/tmp/read_{query}.sql", 4, args.seconds, defines)}
reads.append(row)
print("read ", row, flush=True)
wal_rows = []
for variant in WRITE_VARIANTS:
meta["sizes"][variant] = use_variant(variant)
# (a) throughput/latency under concurrency: 8 clients, time-based, autovacuum as in production
for rep in range(1, args.write_repeats + 1):
pgbench("/tmp/write.sql", 8, args.warmup, {})
row = {"variant": variant, "repeat": rep, **pgbench("/tmp/write.sql", 8, args.seconds, {})}
writes.append(row)
print("write", row, flush=True)
# (b) WAL volume per insert, deterministic: the same 5 000 app transactions (1 client),
# autovacuum off for the table and a CHECKPOINT right before, so every run starts from
# the same state and only the index set differs.
psql("ALTER TABLE messages SET (autovacuum_enabled = off);")
for rep in range(1, 4):
psql("CHECKPOINT;")
lsn0 = psql("SELECT pg_current_wal_lsn();")
sh(["docker", "exec", CLIENT, "pgbench", "-h", "127.0.0.1", "-U", "postgres", "-n", "-M", "prepared",
"-c", "1", "-t", "5000", "-f", "/tmp/write.sql", "bench"])
wal = int(psql(f"SELECT pg_current_wal_lsn() - '{lsn0}'::pg_lsn;"))
wal_row = {"variant": variant, "repeat": rep, "inserts": 5000, "wal_bytes": wal,
"wal_bytes_per_insert": round(wal / 5000)}
wal_rows.append(wal_row)
print("wal ", wal_row, flush=True)
psql("ALTER TABLE messages RESET (autovacuum_enabled);")
sh(["docker", "rm", "-f", "-v", CLIENT, NAME], check=False)
with (out / "wal.csv").open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=list(wal_rows[0]))
writer.writeheader()
writer.writerows(wal_rows)
meta["pgbench"]["write_repeats"] = args.write_repeats
meta["pgbench"]["wal_method"] = "5000 app transactions, 1 client, autovacuum off, CHECKPOINT before; 3 repeats"
for name, rows in (("reads", reads), ("writes", writes)):
if not rows:
continue
with (out / f"{name}.csv").open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
(out / "plans-meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print("done:", out)
return 0
if __name__ == "__main__":
sys.exit(main())