Files
mtusi2026lab2/research/track2_plans_backup.py
T

218 lines
13 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Трек 2 (план запроса и восстановление), lab2.
Что делает (стенд `docker compose up -d` должен работать; рабочие данные не затрагиваются —
используется отдельная БД `research` внутри того же PostgreSQL):
1. Создаёт БД research со схемой из migrations/ и ≥100 000 сообщений с неравномерными размерами чатов
(1 чат 60 000, 5 по 5 000, 100 по 100, 1 000 по 10).
2. Сравнивает 4 варианта индексов и 3 запроса истории: EXPLAIN (ANALYZE, BUFFERS), медиана 9 прогонов.
3. Измеряет стоимость записи для каждого варианта индексов (bulk INSERT и 5 000 одиночных INSERT) и размер индексов.
4. Backup (pg_dump -Fc) -> восстановление в НОВОЕ хранилище (отдельный контейнер postgres с новым volume)
-> сверка количества строк, контрольной суммы всех сообщений и выбранных страниц истории.
5. Показывает границу потери данных: строки, записанные после backup, в восстановленной копии отсутствуют.
Результаты: evidence/track2_*.{json,txt}.
"""
import json
import os
import re
import statistics
import subprocess
import sys
import time
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
OUT = os.path.join(ROOT, "evidence")
TARGET = "mtusi-lab2-restore-target"
TARGET_VOLUME = "mtusi-lab2-restore-target-data"
def sh(args, stdin=None, check=True):
r = subprocess.run(args, cwd=ROOT, input=stdin, capture_output=True, text=True, encoding="utf-8")
if check and r.returncode != 0:
raise RuntimeError("команда %s завершилась с %d: %s" % (args[:4], r.returncode, r.stderr[-800:]))
return r.stdout
def psql(sql, db="research", container=None, tuples=True):
base = (["docker", "exec", "-i", container] if container else
["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres"])
args = base + ["psql", "-X", "-U", "messenger", "-d", db, "-v", "ON_ERROR_STOP=1"] + (["-At"] if tuples else [])
return sh(args, stdin=sql).strip()
def timed_exec(sql):
t = time.perf_counter()
psql(sql)
return time.perf_counter() - t
def setup():
psql("DROP DATABASE IF EXISTS research", db="postgres")
psql("CREATE DATABASE research", db="postgres")
with open(os.path.join(ROOT, "migrations", "0001_init.sql"), encoding="utf-8") as fh:
psql(fh.read())
psql("""
INSERT INTO users (id, username, display_name, password_hash)
SELECT gen_random_uuid(), 'r_' || g, 'user ' || g, 'x' FROM generate_series(1, 200) g;
CREATE TABLE sizes AS
SELECT gen_random_uuid() AS id, n, sz, now() - interval '1 day' + n * interval '1 millisecond' AS created_at,
(SELECT id FROM users ORDER BY random() LIMIT 1) AS u1
FROM (SELECT 1 AS n, 60000 AS sz
UNION ALL SELECT g, 5000 FROM generate_series(2, 6) g
UNION ALL SELECT g, 100 FROM generate_series(7, 106) g
UNION ALL SELECT g, 10 FROM generate_series(107, 1106) g) s;
INSERT INTO chats (id, title, created_at) SELECT id, 'chat ' || n, created_at FROM sizes;
INSERT INTO chat_members (chat_id, user_id, chat_created_at, ord) SELECT id, u1, created_at, 1 FROM sizes;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), s.id, s.u1, 'message ' || i || ' ' || repeat('x', (random() * 200)::int),
s.created_at + i * interval '1 second' + (random() * 999)::int * interval '1 microsecond'
FROM sizes s, generate_series(1, s.sz) i;
ANALYZE;""")
total = psql("SELECT count(*) FROM messages")
big = psql("SELECT id FROM sizes WHERE n = 1")
small = psql("SELECT id FROM sizes WHERE n = 500")
return int(total), big, small
VARIANTS = {
"V0_pk_only": [],
"V1_chat_id": ["CREATE INDEX ix_chat ON messages (chat_id)"],
"V2_chat_created_id": ["CREATE INDEX ix_hist ON messages (chat_id, created_at, id)"],
"V3_chat_created_id_include": ["CREATE INDEX ix_hist_inc ON messages (chat_id, created_at, id) INCLUDE (sender_id)"],
}
def apply_variant(name):
rows = psql("SELECT indexname FROM pg_indexes WHERE tablename = 'messages' AND indexname <> 'messages_pkey'").split()
drops = "".join("DROP INDEX IF EXISTS %s;" % r for r in rows)
psql(drops + ";".join(VARIANTS[name]) + (";" if VARIANTS[name] else "") + "ANALYZE messages;")
def explain(sql, runs=9):
times, text = [], ""
for i in range(runs):
out = psql("EXPLAIN (ANALYZE, BUFFERS) " + sql, tuples=False)
m = re.search(r"Execution Time: ([0-9.]+) ms", out)
times.append(float(m.group(1)))
text = out
node = re.search(r"->\s+([A-Za-z ]+?)(?: using| on| \()", text) or re.search(r"^\s*([A-Za-z ]+?)(?: using| on| \()", text, re.M)
return {"median_ms": round(statistics.median(times), 3), "min_ms": min(times), "plan": text, "top_node": node.group(1).strip() if node else "?"}
def queries(big, small):
mid = psql("SELECT created_at || '|' || id FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET 30000 LIMIT 1" % big)
ts, mid_id = mid.split("|")
return {
"keyset_big_chat_middle": "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' AND (created_at, id) > ('%s', '%s') ORDER BY created_at, id LIMIT 51" % (big, ts, mid_id),
"first_page_big_chat": "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id LIMIT 51" % big,
"offset_big_chat_30000": "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET 30000 LIMIT 51" % big,
"first_page_small_chat": "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id LIMIT 51" % small,
}
def write_cost(name):
"""bulk INSERT всех строк в пустую копию с тем же набором индексов + 5 000 одиночных INSERT."""
psql("DROP TABLE IF EXISTS m_copy; CREATE TABLE m_copy (LIKE messages INCLUDING DEFAULTS INCLUDING CONSTRAINTS);"
"ALTER TABLE m_copy ADD PRIMARY KEY (id);" + ";".join(s.replace("messages", "m_copy").replace("ix_", "cx_") for s in VARIANTS[name]) + ";")
bulk = timed_exec("INSERT INTO m_copy SELECT * FROM messages;")
return bulk
def single_insert_seconds(name):
out = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-X", "-U", "messenger", "-d", "research"],
cwd=ROOT, input="""DO $$ DECLARE t0 timestamptz; c uuid; u uuid; BEGIN
SELECT id, u1 INTO c, u FROM sizes WHERE n = 2; t0 := clock_timestamp();
FOR i IN 1..5000 LOOP INSERT INTO m_copy (id, chat_id, sender_id, text) VALUES (gen_random_uuid(), c, u, 'w' || i); END LOOP;
RAISE NOTICE 'ELAPSED %', extract(epoch FROM clock_timestamp() - t0); END $$;""", capture_output=True, text=True, encoding="utf-8")
m = re.search(r"ELAPSED ([0-9.]+)", out.stderr + out.stdout)
return float(m.group(1)) if m else None
def sizes():
return psql("SELECT pg_size_pretty(pg_relation_size('m_copy')) || ' / indexes ' || pg_size_pretty(pg_indexes_size('m_copy'))")
CHECKSUM_SQL = ("SELECT count(*) || ' ' || md5(string_agg(id::text || chat_id::text || sender_id::text || text || created_at::text, ',' ORDER BY id)) FROM messages")
ROWS_SQL = "SELECT 'users=' || (SELECT count(*) FROM users) || ' chats=' || (SELECT count(*) FROM chats) || ' members=' || (SELECT count(*) FROM chat_members)"
def checksum(db="research", container=None):
return psql(CHECKSUM_SQL, db=db, container=container)
def backup_restore(big):
apply_variant("V2_chat_created_id")
before = {"messages_checksum": checksum(), "rows": psql(ROWS_SQL)}
t = time.perf_counter()
sh(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "sh", "-c", "pg_dump -U messenger -Fc -d research -f /tmp/research.dump"])
backup_s = time.perf_counter() - t
size = int(sh(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "sh", "-c", "stat -c %s /tmp/research.dump"]).strip())
# данные, записанные ПОСЛЕ backup (граница потери: RPO = интервал между backup)
psql("INSERT INTO messages (id, chat_id, sender_id, text) SELECT gen_random_uuid(), id, u1, 'after-backup ' || g FROM sizes, generate_series(1, 25) g WHERE n = 3;")
after_write = checksum()
# новое хранилище
sh(["docker", "rm", "-f", TARGET], check=False)
sh(["docker", "volume", "rm", "-f", TARGET_VOLUME], check=False)
pg_image = "postgres:17.11-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24"
sh(["docker", "run", "-d", "--name", TARGET, "-e", "POSTGRES_USER=messenger", "-e", "POSTGRES_PASSWORD=restore-target-local",
"-e", "POSTGRES_DB=restored", "-v", TARGET_VOLUME + ":/var/lib/postgresql/data", pg_image])
for _ in range(60):
if subprocess.run(["docker", "exec", TARGET, "pg_isready", "-U", "messenger", "-d", "restored"], capture_output=True).returncode == 0:
time.sleep(1)
break
time.sleep(1)
sh(["docker", "compose", "--env-file", ".env", "cp", "postgres:/tmp/research.dump", os.path.join(OUT, ".research.dump")])
sh(["docker", "cp", os.path.join(OUT, ".research.dump"), TARGET + ":/tmp/research.dump"])
t = time.perf_counter()
sh(["docker", "exec", TARGET, "pg_restore", "-U", "messenger", "-d", "restored", "--no-owner", "--exit-on-error", "/tmp/research.dump"])
restore_s = time.perf_counter() - t
os.remove(os.path.join(OUT, ".research.dump"))
restored = {"messages_checksum": checksum("restored", TARGET), "rows": psql(ROWS_SQL, db="restored", container=TARGET)}
page_src = psql("SELECT md5(string_agg(id::text, ',' ORDER BY created_at, id)) FROM (SELECT id, created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET 30000 LIMIT 100) p" % big)
page_dst = psql("SELECT md5(string_agg(id::text, ',' ORDER BY created_at, id)) FROM (SELECT id, created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET 30000 LIMIT 100) p" % big, db="restored", container=TARGET)
indexes = psql("SELECT string_agg(indexname, ',' ORDER BY indexname) FROM pg_indexes WHERE tablename = 'messages'", db="restored", container=TARGET)
sh(["docker", "rm", "-f", TARGET], check=False)
sh(["docker", "volume", "rm", "-f", TARGET_VOLUME], check=False)
return {"backup_seconds": round(backup_s, 2), "dump_bytes": size, "restore_seconds": round(restore_s, 2),
"source_before_backup": before, "source_after_extra_writes": after_write, "restored": restored,
"restore_matches_backup_state": before["messages_checksum"] == restored["messages_checksum"] and before["rows"] == restored["rows"],
"page_checksum_source": page_src, "page_checksum_restored": page_dst, "sample_page_matches": page_src == page_dst,
"extra_rows_written_after_backup_absent_in_restore": after_write != restored["messages_checksum"],
"restored_indexes": indexes}
def main():
os.makedirs(OUT, exist_ok=True)
total, big, small = setup()
print("сообщений:", total)
qs = queries(big, small)
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "messages": total, "postgres": psql("SHOW server_version", db="postgres"),
"chat_sizes": "1x60000, 5x5000, 100x100, 1000x10", "variants": {}}
plans = []
for name in VARIANTS:
apply_variant(name)
entry = {"queries": {}}
for qname, sql in qs.items():
r = explain(sql)
entry["queries"][qname] = {"median_ms": r["median_ms"], "min_ms": r["min_ms"], "top_node": r["top_node"]}
plans.append("===== %s / %s (медиана %s мс из 9) =====\n%s\n" % (name, qname, r["median_ms"], r["plan"]))
report["variants"][name] = entry
# стоимость записи
for name in VARIANTS:
bulk = write_cost(name)
entry = report["variants"][name]
entry["bulk_insert_all_rows_seconds"] = round(bulk, 2)
entry["single_insert_5000_seconds"] = single_insert_seconds(name)
entry["table_and_index_size"] = sizes()
print(name, entry["bulk_insert_all_rows_seconds"], entry["single_insert_5000_seconds"], entry["table_and_index_size"])
report["backup_restore"] = backup_restore(big)
with open(os.path.join(OUT, "track2_results.json"), "w", encoding="utf-8") as fh:
json.dump(report, fh, ensure_ascii=False, indent=2)
with open(os.path.join(OUT, "track2_explain_plans.txt"), "w", encoding="utf-8") as fh:
fh.write("\n".join(plans))
print(json.dumps(report, ensure_ascii=False, indent=2)[:6000])
if __name__ == "__main__":
main()