forked from mtusi2026/mtusi2026lab2
218 lines
13 KiB
Python
218 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""Трек 2 (план запроса и восстановление), lab2.
|
||
|
||
Что делает (стенд `docker compose up -d` должен работать; рабочие данные не затрагиваются —
|
||
используется отдельная БД `research` внутри того же PostgreSQL):
|
||
1. Создаёт БД research со схемой из migrations/ и ≥100 000 сообщений с неравномерными размерами чатов
|
||
(1 чат 60 000, 5 по 5 000, 100 по 100, 1 000 по 10).
|
||
2. Сравнивает 4 варианта индексов и 3 запроса истории: EXPLAIN (ANALYZE, BUFFERS), медиана 9 прогонов.
|
||
3. Измеряет стоимость записи для каждого варианта индексов (bulk INSERT и 5 000 одиночных INSERT) и размер индексов.
|
||
4. Backup (pg_dump -Fc) -> восстановление в НОВОЕ хранилище (отдельный контейнер postgres с новым volume)
|
||
-> сверка количества строк, контрольной суммы всех сообщений и выбранных страниц истории.
|
||
5. Показывает границу потери данных: строки, записанные после backup, в восстановленной копии отсутствуют.
|
||
Результаты: evidence/track2_*.{json,txt}.
|
||
"""
|
||
import json
|
||
import os
|
||
import re
|
||
import statistics
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
|
||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||
OUT = os.path.join(ROOT, "evidence")
|
||
TARGET = "mtusi-lab2-restore-target"
|
||
TARGET_VOLUME = "mtusi-lab2-restore-target-data"
|
||
|
||
|
||
def sh(args, stdin=None, check=True):
|
||
r = subprocess.run(args, cwd=ROOT, input=stdin, capture_output=True, text=True, encoding="utf-8")
|
||
if check and r.returncode != 0:
|
||
raise RuntimeError("команда %s завершилась с %d: %s" % (args[:4], r.returncode, r.stderr[-800:]))
|
||
return r.stdout
|
||
|
||
|
||
def psql(sql, db="research", container=None, tuples=True):
|
||
base = (["docker", "exec", "-i", container] if container else
|
||
["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres"])
|
||
args = base + ["psql", "-X", "-U", "messenger", "-d", db, "-v", "ON_ERROR_STOP=1"] + (["-At"] if tuples else [])
|
||
return sh(args, stdin=sql).strip()
|
||
|
||
|
||
def timed_exec(sql):
|
||
t = time.perf_counter()
|
||
psql(sql)
|
||
return time.perf_counter() - t
|
||
|
||
|
||
def setup():
|
||
psql("DROP DATABASE IF EXISTS research", db="postgres")
|
||
psql("CREATE DATABASE research", db="postgres")
|
||
with open(os.path.join(ROOT, "migrations", "0001_init.sql"), encoding="utf-8") as fh:
|
||
psql(fh.read())
|
||
psql("""
|
||
INSERT INTO users (id, username, display_name, password_hash)
|
||
SELECT gen_random_uuid(), 'r_' || g, 'user ' || g, 'x' FROM generate_series(1, 200) g;
|
||
CREATE TABLE sizes AS
|
||
SELECT gen_random_uuid() AS id, n, sz, now() - interval '1 day' + n * interval '1 millisecond' AS created_at,
|
||
(SELECT id FROM users ORDER BY random() LIMIT 1) AS u1
|
||
FROM (SELECT 1 AS n, 60000 AS sz
|
||
UNION ALL SELECT g, 5000 FROM generate_series(2, 6) g
|
||
UNION ALL SELECT g, 100 FROM generate_series(7, 106) g
|
||
UNION ALL SELECT g, 10 FROM generate_series(107, 1106) g) s;
|
||
INSERT INTO chats (id, title, created_at) SELECT id, 'chat ' || n, created_at FROM sizes;
|
||
INSERT INTO chat_members (chat_id, user_id, chat_created_at, ord) SELECT id, u1, created_at, 1 FROM sizes;
|
||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||
SELECT gen_random_uuid(), s.id, s.u1, 'message ' || i || ' ' || repeat('x', (random() * 200)::int),
|
||
s.created_at + i * interval '1 second' + (random() * 999)::int * interval '1 microsecond'
|
||
FROM sizes s, generate_series(1, s.sz) i;
|
||
ANALYZE;""")
|
||
total = psql("SELECT count(*) FROM messages")
|
||
big = psql("SELECT id FROM sizes WHERE n = 1")
|
||
small = psql("SELECT id FROM sizes WHERE n = 500")
|
||
return int(total), big, small
|
||
|
||
|
||
VARIANTS = {
|
||
"V0_pk_only": [],
|
||
"V1_chat_id": ["CREATE INDEX ix_chat ON messages (chat_id)"],
|
||
"V2_chat_created_id": ["CREATE INDEX ix_hist ON messages (chat_id, created_at, id)"],
|
||
"V3_chat_created_id_include": ["CREATE INDEX ix_hist_inc ON messages (chat_id, created_at, id) INCLUDE (sender_id)"],
|
||
}
|
||
|
||
|
||
def apply_variant(name):
|
||
rows = psql("SELECT indexname FROM pg_indexes WHERE tablename = 'messages' AND indexname <> 'messages_pkey'").split()
|
||
drops = "".join("DROP INDEX IF EXISTS %s;" % r for r in rows)
|
||
psql(drops + ";".join(VARIANTS[name]) + (";" if VARIANTS[name] else "") + "ANALYZE messages;")
|
||
|
||
|
||
def explain(sql, runs=9):
|
||
times, text = [], ""
|
||
for i in range(runs):
|
||
out = psql("EXPLAIN (ANALYZE, BUFFERS) " + sql, tuples=False)
|
||
m = re.search(r"Execution Time: ([0-9.]+) ms", out)
|
||
times.append(float(m.group(1)))
|
||
text = out
|
||
node = re.search(r"->\s+([A-Za-z ]+?)(?: using| on| \()", text) or re.search(r"^\s*([A-Za-z ]+?)(?: using| on| \()", text, re.M)
|
||
return {"median_ms": round(statistics.median(times), 3), "min_ms": min(times), "plan": text, "top_node": node.group(1).strip() if node else "?"}
|
||
|
||
|
||
def queries(big, small):
|
||
mid = psql("SELECT created_at || '|' || id FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET 30000 LIMIT 1" % big)
|
||
ts, mid_id = mid.split("|")
|
||
return {
|
||
"keyset_big_chat_middle": "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' AND (created_at, id) > ('%s', '%s') ORDER BY created_at, id LIMIT 51" % (big, ts, mid_id),
|
||
"first_page_big_chat": "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id LIMIT 51" % big,
|
||
"offset_big_chat_30000": "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET 30000 LIMIT 51" % big,
|
||
"first_page_small_chat": "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id LIMIT 51" % small,
|
||
}
|
||
|
||
|
||
def write_cost(name):
|
||
"""bulk INSERT всех строк в пустую копию с тем же набором индексов + 5 000 одиночных INSERT."""
|
||
psql("DROP TABLE IF EXISTS m_copy; CREATE TABLE m_copy (LIKE messages INCLUDING DEFAULTS INCLUDING CONSTRAINTS);"
|
||
"ALTER TABLE m_copy ADD PRIMARY KEY (id);" + ";".join(s.replace("messages", "m_copy").replace("ix_", "cx_") for s in VARIANTS[name]) + ";")
|
||
bulk = timed_exec("INSERT INTO m_copy SELECT * FROM messages;")
|
||
return bulk
|
||
|
||
|
||
def single_insert_seconds(name):
|
||
out = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-X", "-U", "messenger", "-d", "research"],
|
||
cwd=ROOT, input="""DO $$ DECLARE t0 timestamptz; c uuid; u uuid; BEGIN
|
||
SELECT id, u1 INTO c, u FROM sizes WHERE n = 2; t0 := clock_timestamp();
|
||
FOR i IN 1..5000 LOOP INSERT INTO m_copy (id, chat_id, sender_id, text) VALUES (gen_random_uuid(), c, u, 'w' || i); END LOOP;
|
||
RAISE NOTICE 'ELAPSED %', extract(epoch FROM clock_timestamp() - t0); END $$;""", capture_output=True, text=True, encoding="utf-8")
|
||
m = re.search(r"ELAPSED ([0-9.]+)", out.stderr + out.stdout)
|
||
return float(m.group(1)) if m else None
|
||
|
||
|
||
def sizes():
|
||
return psql("SELECT pg_size_pretty(pg_relation_size('m_copy')) || ' / indexes ' || pg_size_pretty(pg_indexes_size('m_copy'))")
|
||
|
||
|
||
CHECKSUM_SQL = ("SELECT count(*) || ' ' || md5(string_agg(id::text || chat_id::text || sender_id::text || text || created_at::text, ',' ORDER BY id)) FROM messages")
|
||
ROWS_SQL = "SELECT 'users=' || (SELECT count(*) FROM users) || ' chats=' || (SELECT count(*) FROM chats) || ' members=' || (SELECT count(*) FROM chat_members)"
|
||
|
||
|
||
def checksum(db="research", container=None):
|
||
return psql(CHECKSUM_SQL, db=db, container=container)
|
||
|
||
|
||
def backup_restore(big):
|
||
apply_variant("V2_chat_created_id")
|
||
before = {"messages_checksum": checksum(), "rows": psql(ROWS_SQL)}
|
||
t = time.perf_counter()
|
||
sh(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "sh", "-c", "pg_dump -U messenger -Fc -d research -f /tmp/research.dump"])
|
||
backup_s = time.perf_counter() - t
|
||
size = int(sh(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "sh", "-c", "stat -c %s /tmp/research.dump"]).strip())
|
||
# данные, записанные ПОСЛЕ backup (граница потери: RPO = интервал между backup)
|
||
psql("INSERT INTO messages (id, chat_id, sender_id, text) SELECT gen_random_uuid(), id, u1, 'after-backup ' || g FROM sizes, generate_series(1, 25) g WHERE n = 3;")
|
||
after_write = checksum()
|
||
# новое хранилище
|
||
sh(["docker", "rm", "-f", TARGET], check=False)
|
||
sh(["docker", "volume", "rm", "-f", TARGET_VOLUME], check=False)
|
||
pg_image = "postgres:17.11-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24"
|
||
sh(["docker", "run", "-d", "--name", TARGET, "-e", "POSTGRES_USER=messenger", "-e", "POSTGRES_PASSWORD=restore-target-local",
|
||
"-e", "POSTGRES_DB=restored", "-v", TARGET_VOLUME + ":/var/lib/postgresql/data", pg_image])
|
||
for _ in range(60):
|
||
if subprocess.run(["docker", "exec", TARGET, "pg_isready", "-U", "messenger", "-d", "restored"], capture_output=True).returncode == 0:
|
||
time.sleep(1)
|
||
break
|
||
time.sleep(1)
|
||
sh(["docker", "compose", "--env-file", ".env", "cp", "postgres:/tmp/research.dump", os.path.join(OUT, ".research.dump")])
|
||
sh(["docker", "cp", os.path.join(OUT, ".research.dump"), TARGET + ":/tmp/research.dump"])
|
||
t = time.perf_counter()
|
||
sh(["docker", "exec", TARGET, "pg_restore", "-U", "messenger", "-d", "restored", "--no-owner", "--exit-on-error", "/tmp/research.dump"])
|
||
restore_s = time.perf_counter() - t
|
||
os.remove(os.path.join(OUT, ".research.dump"))
|
||
restored = {"messages_checksum": checksum("restored", TARGET), "rows": psql(ROWS_SQL, db="restored", container=TARGET)}
|
||
page_src = psql("SELECT md5(string_agg(id::text, ',' ORDER BY created_at, id)) FROM (SELECT id, created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET 30000 LIMIT 100) p" % big)
|
||
page_dst = psql("SELECT md5(string_agg(id::text, ',' ORDER BY created_at, id)) FROM (SELECT id, created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET 30000 LIMIT 100) p" % big, db="restored", container=TARGET)
|
||
indexes = psql("SELECT string_agg(indexname, ',' ORDER BY indexname) FROM pg_indexes WHERE tablename = 'messages'", db="restored", container=TARGET)
|
||
sh(["docker", "rm", "-f", TARGET], check=False)
|
||
sh(["docker", "volume", "rm", "-f", TARGET_VOLUME], check=False)
|
||
return {"backup_seconds": round(backup_s, 2), "dump_bytes": size, "restore_seconds": round(restore_s, 2),
|
||
"source_before_backup": before, "source_after_extra_writes": after_write, "restored": restored,
|
||
"restore_matches_backup_state": before["messages_checksum"] == restored["messages_checksum"] and before["rows"] == restored["rows"],
|
||
"page_checksum_source": page_src, "page_checksum_restored": page_dst, "sample_page_matches": page_src == page_dst,
|
||
"extra_rows_written_after_backup_absent_in_restore": after_write != restored["messages_checksum"],
|
||
"restored_indexes": indexes}
|
||
|
||
|
||
def main():
|
||
os.makedirs(OUT, exist_ok=True)
|
||
total, big, small = setup()
|
||
print("сообщений:", total)
|
||
qs = queries(big, small)
|
||
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "messages": total, "postgres": psql("SHOW server_version", db="postgres"),
|
||
"chat_sizes": "1x60000, 5x5000, 100x100, 1000x10", "variants": {}}
|
||
plans = []
|
||
for name in VARIANTS:
|
||
apply_variant(name)
|
||
entry = {"queries": {}}
|
||
for qname, sql in qs.items():
|
||
r = explain(sql)
|
||
entry["queries"][qname] = {"median_ms": r["median_ms"], "min_ms": r["min_ms"], "top_node": r["top_node"]}
|
||
plans.append("===== %s / %s (медиана %s мс из 9) =====\n%s\n" % (name, qname, r["median_ms"], r["plan"]))
|
||
report["variants"][name] = entry
|
||
# стоимость записи
|
||
for name in VARIANTS:
|
||
bulk = write_cost(name)
|
||
entry = report["variants"][name]
|
||
entry["bulk_insert_all_rows_seconds"] = round(bulk, 2)
|
||
entry["single_insert_5000_seconds"] = single_insert_seconds(name)
|
||
entry["table_and_index_size"] = sizes()
|
||
print(name, entry["bulk_insert_all_rows_seconds"], entry["single_insert_5000_seconds"], entry["table_and_index_size"])
|
||
report["backup_restore"] = backup_restore(big)
|
||
with open(os.path.join(OUT, "track2_results.json"), "w", encoding="utf-8") as fh:
|
||
json.dump(report, fh, ensure_ascii=False, indent=2)
|
||
with open(os.path.join(OUT, "track2_explain_plans.txt"), "w", encoding="utf-8") as fh:
|
||
fh.write("\n".join(plans))
|
||
print(json.dumps(report, ensure_ascii=False, indent=2)[:6000])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|