forked from mtusi2026/mtusi2026lab2
plans.py: 121 120 сообщений (60k/15k/2k/80), 3 индекса x keyset/OFFSET x 3 размера чата, pgbench в отдельном контейнере; WAL на вставку детерминированно. backup_restore.py: pg_dump живой БД во время записи через API, восстановление в новый контейнер и volume, сверка MD5 по чатам, API на восстановленной БД. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
287 lines
15 KiB
Python
287 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Track 2, part 1: query plans and write cost on >= 100 000 messages with uneven chat sizes.
|
|
|
|
python bench/track2/plans.py # ~20 min, results in evidence/track2/
|
|
|
|
An ISOLATED PostgreSQL container (same pinned image as compose.yaml, --cpus 2 --memory 2g,
|
|
no published ports) gets the schema from migrations/*.sql and a deterministic synthetic
|
|
dataset: 1 chat x 60 000, 2 x 15 000, 8 x 2 000, 189 x 80 messages = 121 120 messages,
|
|
texts of 10..400 bytes. The live stand is not touched: indexes are dropped and created
|
|
here freely.
|
|
|
|
For every index variant the script runs pgbench (prepared statements, like the driver):
|
|
* reads — one history page (LIMIT 51) after a RANDOM position of a big/medium/small chat,
|
|
keyset query (the app's SQL) vs OFFSET; 4 clients;
|
|
* writes — the app's message transaction (UPDATE chats ... RETURNING + INSERT); 8 clients;
|
|
and records TPS, p50/p95/p99 latency, WAL bytes per insert, index/table sizes, EXPLAIN.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import json
|
|
import math
|
|
import platform
|
|
import secrets
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "src"))
|
|
from messenger.repo import HISTORY_AFTER # noqa: E402 (the app's real statement)
|
|
|
|
IMAGE = "postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c"
|
|
NAME = "mtusi-track2-pg"
|
|
CLIENT = "mtusi-track2-client" # pgbench runs here: it must not eat the server's CPU quota
|
|
PASSWORD = secrets.token_urlsafe(16) # throwaway, never printed
|
|
USER_ID = "00000000-0000-4000-8000-0000000b0001"
|
|
|
|
VARIANTS = {
|
|
# name: (DDL of secondary indexes on messages, description)
|
|
"keyset_index": (["CREATE INDEX messages_chat_keyset ON messages (chat_id, created_at, id)"],
|
|
"(chat_id, created_at, id) — used by the app"),
|
|
"chat_id_index": (["CREATE INDEX messages_chat_only ON messages (chat_id)"],
|
|
"(chat_id) only"),
|
|
"no_index": ([], "primary key only"),
|
|
"keyset_plus_sender": (["CREATE INDEX messages_chat_keyset ON messages (chat_id, created_at, id)",
|
|
"CREATE INDEX messages_sender_time ON messages (sender_id, created_at)"],
|
|
"app index + a second index (sender_id, created_at)"),
|
|
}
|
|
READ_VARIANTS = ["keyset_index", "chat_id_index", "no_index"]
|
|
WRITE_VARIANTS = ["no_index", "keyset_index", "keyset_plus_sender"]
|
|
KINDS = {"big": 60_000, "medium": 15_000, "small": 80}
|
|
|
|
|
|
def sh(args: list[str], input_text: str | None = None, check: bool = True) -> str:
|
|
result = subprocess.run(args, input=input_text, capture_output=True, text=True, encoding="utf-8")
|
|
if check and result.returncode != 0:
|
|
raise RuntimeError(f"{' '.join(args[:4])}: {result.stderr.strip()[-1500:]}")
|
|
return result.stdout
|
|
|
|
|
|
def psql(sql: str) -> str:
|
|
return sh(["docker", "exec", "-i", NAME, "psql", "-U", "postgres", "-d", "bench", "-X", "-q", "-At",
|
|
"-v", "ON_ERROR_STOP=1", "-f", "-"], input_text=sql).strip()
|
|
|
|
|
|
def put(path: str, content: str) -> None:
|
|
sh(["docker", "exec", "-i", CLIENT, "sh", "-c", f"cat > {path}"], input_text=content)
|
|
|
|
|
|
def start_container(cpus: str, memory: str) -> None:
|
|
sh(["docker", "rm", "-f", "-v", CLIENT, NAME], check=False)
|
|
sh(["docker", "run", "-d", "--name", NAME, "--cpus", cpus, "--memory", memory,
|
|
"-e", f"POSTGRES_PASSWORD={PASSWORD}", "-e", "POSTGRES_DB=bench", IMAGE])
|
|
# Load generator: separate container (own CPU quota) sharing the server's network namespace.
|
|
sh(["docker", "run", "-d", "--name", CLIENT, "--network", f"container:{NAME}", "--cpus", "2",
|
|
"-e", f"PGPASSWORD={PASSWORD}", "--entrypoint", "sleep", IMAGE, "infinity"])
|
|
for _ in range(120):
|
|
if subprocess.run(["docker", "exec", NAME, "pg_isready", "-U", "postgres", "-d", "bench", "-q"]).returncode == 0:
|
|
time.sleep(1)
|
|
return
|
|
time.sleep(0.5)
|
|
raise RuntimeError("experiment PostgreSQL did not start")
|
|
|
|
|
|
SEED = f"""
|
|
SELECT setseed(0.42);
|
|
INSERT INTO users (id, username, display_name, password_hash)
|
|
VALUES ('{USER_ID}', 'track2_bench', convert_to('Track 2', 'UTF8'), 'not-a-login');
|
|
CREATE TABLE bench_sizes AS
|
|
SELECT n, CASE WHEN n = 1 THEN 60000 WHEN n <= 3 THEN 15000 WHEN n <= 11 THEN 2000 ELSE 80 END AS size,
|
|
md5('track2-chat-' || n)::uuid AS chat_id
|
|
FROM generate_series(1::int, 200::int) AS n;
|
|
INSERT INTO chats (id, title, created_by, created_at)
|
|
SELECT chat_id, convert_to('chat ' || n, 'UTF8'), '{USER_ID}', timestamptz '2026-01-01 00:00:00+00' FROM bench_sizes;
|
|
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
|
|
SELECT chat_id, '{USER_ID}', 0, timestamptz '2026-01-01 00:00:00+00' FROM bench_sizes;
|
|
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
|
SELECT gen_random_uuid(), s.chat_id, '{USER_ID}',
|
|
convert_to(left(repeat('сообщение ', 40), 5 + (random() * 195)::int), 'UTF8'),
|
|
timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
|
|
FROM bench_sizes s CROSS JOIN LATERAL generate_series(1::int, s.size) AS g;
|
|
UPDATE chats c SET last_message_at = m.mx FROM (SELECT chat_id, max(created_at) mx FROM messages GROUP BY chat_id) m
|
|
WHERE m.chat_id = c.id;
|
|
CREATE TABLE bench_chats AS SELECT n, chat_id AS id FROM bench_sizes;
|
|
CREATE TABLE bench_cursors AS
|
|
SELECT kind, row_number() OVER (PARTITION BY kind ORDER BY rn)::int AS n, created_at AS ts, id AS mid, rn::int AS off
|
|
FROM (SELECT CASE s.n WHEN 1 THEN 'big' WHEN 2 THEN 'medium' ELSE 'small' END AS kind, m.created_at, m.id,
|
|
row_number() OVER (PARTITION BY m.chat_id ORDER BY m.created_at, m.id) AS rn, s.size
|
|
FROM messages m JOIN bench_sizes s ON s.chat_id = m.chat_id WHERE s.n IN (1, 2, 12)) x
|
|
WHERE rn % greatest(1, size / 200) = 0 AND rn < size;
|
|
CREATE INDEX ON bench_cursors (kind, n);
|
|
VACUUM ANALYZE;
|
|
"""
|
|
|
|
|
|
def pg_param(sql: str) -> str:
|
|
"""psycopg %s placeholders -> pgbench :variables (passed as parameters in prepared mode)."""
|
|
names = iter([":chat", ":cts", ":cid", "51"])
|
|
out = sql
|
|
while "%s" in out:
|
|
out = out.replace("%s", next(names), 1)
|
|
return out
|
|
|
|
|
|
READ_SCRIPTS = {
|
|
"keyset": "\\set k random(1, :ncur)\n"
|
|
"SELECT ts AS cts, mid AS cid FROM bench_cursors WHERE kind = :kind AND n = :k \\gset\n"
|
|
+ pg_param(HISTORY_AFTER) + ";\n",
|
|
"offset": "\\set k random(1, :ncur)\n"
|
|
"SELECT off FROM bench_cursors WHERE kind = :kind AND n = :k \\gset\n"
|
|
"SELECT id, sender_id, text, created_at FROM messages WHERE chat_id = :chat "
|
|
"ORDER BY created_at, id OFFSET :off LIMIT 51;\n",
|
|
}
|
|
WRITE_SCRIPT = (
|
|
"\\set c random(1, 200)\n"
|
|
"SELECT id AS chat FROM bench_chats WHERE n = :c \\gset\n"
|
|
"BEGIN;\n"
|
|
"UPDATE chats SET last_message_at = greatest(clock_timestamp(), last_message_at + interval '1 microsecond') "
|
|
"WHERE id = :chat RETURNING last_message_at AS ts \\gset\n"
|
|
f"INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES "
|
|
f"(gen_random_uuid(), :chat, '{USER_ID}', convert_to(repeat('w', 120), 'UTF8'), :ts);\n"
|
|
"END;\n"
|
|
)
|
|
|
|
|
|
def pgbench(script_path: str, clients: int, seconds: int, defines: dict[str, str]) -> dict:
|
|
"""Run pgbench with per-transaction logging; return TPS and latency percentiles (ms)."""
|
|
sh(["docker", "exec", CLIENT, "sh", "-c", "rm -f /tmp/pgblog*"])
|
|
args = ["docker", "exec", CLIENT, "sh", "-c",
|
|
"cd /tmp && pgbench -h 127.0.0.1 -U postgres -n -M prepared -c {c} -j {c} -T {t} -l --log-prefix=pgblog {d} -f {f} bench".format(
|
|
c=clients, t=seconds, f=script_path, d=" ".join(f"-D {k}={v}" for k, v in defines.items()))]
|
|
out = sh(args)
|
|
tps = float(next(l for l in out.splitlines() if l.startswith("tps = ")).split()[2])
|
|
failed = next((l for l in out.splitlines() if "number of failed transactions" in l), "0").split(":")[-1].split()[0]
|
|
lat = sorted(int(line.split()[2]) / 1000 for line in sh(["docker", "exec", CLIENT, "sh", "-c", "cat /tmp/pgblog*"]).splitlines())
|
|
|
|
def pct(p: float) -> float:
|
|
return round(lat[max(0, math.ceil(p * len(lat)) - 1)], 3)
|
|
return {"tps": round(tps, 1), "transactions": len(lat), "failed": int(failed),
|
|
"p50_ms": pct(.50), "p95_ms": pct(.95), "p99_ms": pct(.99), "mean_ms": round(statistics.mean(lat), 3)}
|
|
|
|
|
|
def use_variant(name: str) -> dict:
|
|
psql("DROP INDEX IF EXISTS messages_chat_keyset; DROP INDEX IF EXISTS messages_chat_only; "
|
|
"DROP INDEX IF EXISTS messages_sender_time;")
|
|
started = time.monotonic()
|
|
for ddl in VARIANTS[name][0]:
|
|
psql(ddl + ";")
|
|
build_s = round(time.monotonic() - started, 2)
|
|
psql("VACUUM ANALYZE messages;")
|
|
sizes = psql("SELECT indexrelname || '=' || pg_relation_size(indexrelid) FROM pg_stat_user_indexes "
|
|
"WHERE relname = 'messages' ORDER BY 1;").split()
|
|
return {"index_build_s": build_s, "index_bytes": dict(s.split("=") for s in sizes),
|
|
"table_bytes": int(psql("SELECT pg_relation_size('messages');"))}
|
|
|
|
|
|
def explain_late(kind: str) -> str:
|
|
chat = psql(f"SELECT chat_id FROM bench_sizes WHERE n = {dict(big=1, medium=2, small=12)[kind]};")
|
|
return psql(f"""
|
|
SET plan_cache_mode = force_generic_plan;
|
|
PREPARE p(uuid, timestamptz, uuid, int) AS {HISTORY_AFTER.replace('%s', '$1', 1).replace('%s', '$2', 1).replace('%s', '$3', 1).replace('%s', '$4', 1)};
|
|
SELECT ts, mid FROM bench_cursors WHERE kind = '{kind}' ORDER BY n DESC LIMIT 1 \\gset
|
|
EXPLAIN (ANALYZE, BUFFERS) EXECUTE p('{chat}', :'ts', :'mid', 51);
|
|
""")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
parser.add_argument("--seconds", type=int, default=10)
|
|
parser.add_argument("--warmup", type=int, default=3)
|
|
parser.add_argument("--repeats", type=int, default=3)
|
|
parser.add_argument("--out", default=str(ROOT / "evidence/track2"))
|
|
parser.add_argument("--writes-only", action="store_true", help="re-measure writes, keep reads.csv")
|
|
parser.add_argument("--write-repeats", type=int, default=5)
|
|
args = parser.parse_args()
|
|
out = Path(args.out)
|
|
(out / "explain").mkdir(parents=True, exist_ok=True)
|
|
|
|
start_container("2", "2g")
|
|
schema = "".join(p.read_text(encoding="utf-8") for p in sorted((ROOT / "migrations").glob("*.sql")))
|
|
psql(schema)
|
|
seed_started = time.monotonic()
|
|
psql(SEED)
|
|
seed_s = round(time.monotonic() - seed_started, 1)
|
|
stats = psql("SELECT count(*), count(DISTINCT chat_id), min(octet_length(text)), max(octet_length(text)) FROM messages;")
|
|
sizes = psql("SELECT size || 'x' || count(*) FROM bench_sizes GROUP BY size ORDER BY size DESC;").split()
|
|
for name, script in READ_SCRIPTS.items():
|
|
put(f"/tmp/read_{name}.sql", script)
|
|
put("/tmp/write.sql", WRITE_SCRIPT)
|
|
chats = {k: psql(f"SELECT chat_id FROM bench_sizes WHERE n = {dict(big=1, medium=2, small=12)[k]};") for k in KINDS}
|
|
ncur = {k: int(psql(f"SELECT count(*) FROM bench_cursors WHERE kind = '{k}';")) for k in KINDS}
|
|
meta = {"image": IMAGE, "limits": {"server_cpus": 2, "server_memory": "2g", "pgbench_client_cpus": 2}, "pg_settings": "image defaults",
|
|
"dataset": {"messages_chats_minbytes_maxbytes": stats, "chat_sizes": sizes, "seed_seconds": seed_s},
|
|
"pgbench": {"mode": "prepared", "seconds": args.seconds, "warmup": args.warmup, "repeats": args.repeats,
|
|
"read_clients": 4, "write_clients": 8, "page": "LIMIT 51 after a random position"},
|
|
"host": {"platform": platform.platform(), "docker": sh(["docker", "version", "-f", "{{.Server.Version}}"]).strip()},
|
|
"variants": {k: v[1] for k, v in VARIANTS.items()}, "sizes": {}}
|
|
print("dataset:", stats, sizes, flush=True)
|
|
|
|
reads, writes = [], []
|
|
if args.writes_only:
|
|
old = json.loads((out / "plans-meta.json").read_text(encoding="utf-8"))
|
|
meta["sizes"] = old.get("sizes", {})
|
|
for variant in ([] if args.writes_only else READ_VARIANTS):
|
|
meta["sizes"][variant] = use_variant(variant)
|
|
for kind in KINDS:
|
|
(out / "explain" / f"{variant}-{kind}-late-page.txt").write_text(explain_late(kind), encoding="utf-8", newline="\n")
|
|
for rep in range(1, args.repeats + 1):
|
|
for query in READ_SCRIPTS:
|
|
for kind in KINDS:
|
|
defines = {"chat": chats[kind], "kind": kind, "ncur": str(ncur[kind])}
|
|
pgbench(f"/tmp/read_{query}.sql", 4, args.warmup, defines) # warm-up, discarded
|
|
row = {"variant": variant, "query": query, "chat": kind, "chat_size": KINDS[kind], "repeat": rep,
|
|
**pgbench(f"/tmp/read_{query}.sql", 4, args.seconds, defines)}
|
|
reads.append(row)
|
|
print("read ", row, flush=True)
|
|
wal_rows = []
|
|
for variant in WRITE_VARIANTS:
|
|
meta["sizes"][variant] = use_variant(variant)
|
|
# (a) throughput/latency under concurrency: 8 clients, time-based, autovacuum as in production
|
|
for rep in range(1, args.write_repeats + 1):
|
|
pgbench("/tmp/write.sql", 8, args.warmup, {})
|
|
row = {"variant": variant, "repeat": rep, **pgbench("/tmp/write.sql", 8, args.seconds, {})}
|
|
writes.append(row)
|
|
print("write", row, flush=True)
|
|
# (b) WAL volume per insert, deterministic: the same 5 000 app transactions (1 client),
|
|
# autovacuum off for the table and a CHECKPOINT right before, so every run starts from
|
|
# the same state and only the index set differs.
|
|
psql("ALTER TABLE messages SET (autovacuum_enabled = off);")
|
|
for rep in range(1, 4):
|
|
psql("CHECKPOINT;")
|
|
lsn0 = psql("SELECT pg_current_wal_lsn();")
|
|
sh(["docker", "exec", CLIENT, "pgbench", "-h", "127.0.0.1", "-U", "postgres", "-n", "-M", "prepared",
|
|
"-c", "1", "-t", "5000", "-f", "/tmp/write.sql", "bench"])
|
|
wal = int(psql(f"SELECT pg_current_wal_lsn() - '{lsn0}'::pg_lsn;"))
|
|
wal_row = {"variant": variant, "repeat": rep, "inserts": 5000, "wal_bytes": wal,
|
|
"wal_bytes_per_insert": round(wal / 5000)}
|
|
wal_rows.append(wal_row)
|
|
print("wal ", wal_row, flush=True)
|
|
psql("ALTER TABLE messages RESET (autovacuum_enabled);")
|
|
sh(["docker", "rm", "-f", "-v", CLIENT, NAME], check=False)
|
|
with (out / "wal.csv").open("w", newline="", encoding="utf-8") as f:
|
|
writer = csv.DictWriter(f, fieldnames=list(wal_rows[0]))
|
|
writer.writeheader()
|
|
writer.writerows(wal_rows)
|
|
|
|
meta["pgbench"]["write_repeats"] = args.write_repeats
|
|
meta["pgbench"]["wal_method"] = "5000 app transactions, 1 client, autovacuum off, CHECKPOINT before; 3 repeats"
|
|
for name, rows in (("reads", reads), ("writes", writes)):
|
|
if not rows:
|
|
continue
|
|
with (out / f"{name}.csv").open("w", newline="", encoding="utf-8") as f:
|
|
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
(out / "plans-meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
print("done:", out)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|