From d3fafb85922a737c40d77127d8f656b44223cea7 Mon Sep 17 00:00:00 2001 From: vasyukova Date: Fri, 2 Oct 2026 15:09:40 +0300 Subject: [PATCH] =?UTF-8?q?=D1=80=D0=B5=D0=B0=D0=BB=D0=B8=D0=B7=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D0=BB=D0=B0=20=D1=83=D1=80=D0=BE=D0=B2=D0=B5=D0=BD?= =?UTF-8?q?=D1=8C=205,=20=D1=82=D1=80=D0=B5=D0=BA=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- evidence/lab2_track2_explain.md | 104 +++++++ evidence/lab2_track2_write_cost.json | 20 ++ student-tests/test_track2.py | 409 +++++++++++++++++++++++++++ student-tests/track2_explain.sql | 10 + student-tests/track2_seed.sql | 79 ++++++ 5 files changed, 622 insertions(+) create mode 100644 evidence/lab2_track2_explain.md create mode 100644 evidence/lab2_track2_write_cost.json create mode 100644 student-tests/test_track2.py create mode 100644 student-tests/track2_explain.sql create mode 100644 student-tests/track2_seed.sql diff --git a/evidence/lab2_track2_explain.md b/evidence/lab2_track2_explain.md new file mode 100644 index 00000000..9ddb612c --- /dev/null +++ b/evidence/lab2_track2_explain.md @@ -0,0 +1,104 @@ +# Track 2: EXPLAIN (ANALYZE, BUFFERS) + +## big chat +``` +Limit (cost=4726.04..4731.85 rows=51 width=69) (actual time=28.316..32.077 rows=51.00 loops=1) + Buffers: shared hit=1639 + -> Gather Merge (cost=4726.04..9161.77 rows=38920 width=69) (actual time=28.313..32.066 rows=51.00 loops=1) + Workers Planned: 1 + Workers Launched: 1 + Buffers: shared hit=1639 + -> Sort (cost=3726.03..3783.26 rows=22894 width=69) (actual time=6.294..6.298 rows=25.50 loops=2) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 38kB + Buffers: shared hit=1639 + Worker 0: Sort Method: quicksort Memory: 25kB + -> Parallel Seq Scan on messages (cost=0.00..2962.24 rows=22894 width=69) (actual time=0.010..4.738 rows=13636.50 loops=2) + Filter: ((chat_id = '79d37065-bfb0-4cb1-8f1e-e79533f724ff'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 15:03:17.148852+03'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) + Rows Removed by Filter: 51364 + Buffers: shared hit=1624 +Planning: + Buffers: shared hit=88 +Planning Time: 1.487 ms +Execution Time: 32.133 ms + +``` + +## small chat +``` +Limit (cost=3899.08..3899.09 rows=6 width=69) (actual time=7.785..7.787 rows=1.00 loops=1) + Buffers: shared hit=1630 + -> Sort (cost=3899.08..3899.09 rows=6 width=69) (actual time=7.783..7.783 rows=1.00 loops=1) + Sort Key: created_at, id + Sort Method: quicksort Memory: 25kB + Buffers: shared hit=1630 + -> Seq Scan on messages (cost=0.00..3899.00 rows=6 width=69) (actual time=0.056..7.764 rows=1.00 loops=1) + Filter: ((chat_id = '730e474b-405e-4558-8cf3-ac6a17c2a7d1'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 01:23:13.456347+03'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) + Rows Removed by Filter: 129999 + Buffers: shared hit=1624 +Planning: + Buffers: shared hit=88 +Planning Time: 1.182 ms +Execution Time: 7.820 ms + +``` + +## no_index +``` +Limit (cost=4475.27..4475.40 rows=51 width=69) (actual time=12.589..12.596 rows=51.00 loops=1) + Buffers: shared hit=729 + -> Sort (cost=4475.27..4572.12 rows=38739 width=69) (actual time=12.587..12.589 rows=51.00 loops=1) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 38kB + Buffers: shared hit=729 + -> Bitmap Heap Scan on messages (cost=609.04..3182.85 rows=38739 width=69) (actual time=0.891..8.050 rows=27273.00 loops=1) + Recheck Cond: (chat_id = '79d37065-bfb0-4cb1-8f1e-e79533f724ff'::uuid) + Filter: (ROW(created_at, id) > ROW('2026-10-02 15:03:17.148852+03'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid)) + Rows Removed by Filter: 27272 + Heap Blocks: exact=675 + Buffers: shared hit=723 + -> Bitmap Index Scan on idx_messages_chat_only (cost=0.00..599.36 rows=54275 width=0) (actual time=0.823..0.823 rows=54545.00 loops=1) + Index Cond: (chat_id = '79d37065-bfb0-4cb1-8f1e-e79533f724ff'::uuid) + Index Searches: 1 + Buffers: shared hit=48 +Planning: + Buffers: shared hit=100 +Planning Time: 2.045 ms +Execution Time: 12.641 ms + +``` + +## chat_only +``` +Limit (cost=23.89..23.91 rows=8 width=69) (actual time=0.037..0.037 rows=1.00 loops=1) + Buffers: shared hit=9 + -> Sort (cost=23.89..23.91 rows=8 width=69) (actual time=0.035..0.035 rows=1.00 loops=1) + Sort Key: created_at, id + Sort Method: quicksort Memory: 25kB + Buffers: shared hit=9 + -> Index Scan using idx_messages_chat_only on messages (cost=0.29..23.77 rows=8 width=69) (actual time=0.015..0.015 rows=1.00 loops=1) + Index Cond: (chat_id = '730e474b-405e-4558-8cf3-ac6a17c2a7d1'::uuid) + Filter: (ROW(created_at, id) > ROW('2026-10-02 01:23:13.456347+03'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid)) + Index Searches: 1 + Buffers: shared hit=3 +Planning: + Buffers: shared hit=100 +Planning Time: 2.240 ms +Execution Time: 0.066 ms + +``` + +## full (chat_id, created_at, id) +``` +Limit (cost=0.42..9.42 rows=51 width=69) (actual time=0.101..0.118 rows=51.00 loops=1) + Buffers: shared hit=5 read=1 + -> Index Scan using idx_messages_chat_created on messages (cost=0.42..6871.59 rows=38939 width=69) (actual time=0.098..0.112 rows=51.00 loops=1) + Index Cond: ((chat_id = '79d37065-bfb0-4cb1-8f1e-e79533f724ff'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 15:03:17.148852+03'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) + Index Searches: 1 + Buffers: shared hit=5 read=1 +Planning: + Buffers: shared hit=111 +Planning Time: 1.697 ms +Execution Time: 0.153 ms + +``` diff --git a/evidence/lab2_track2_write_cost.json b/evidence/lab2_track2_write_cost.json new file mode 100644 index 00000000..95a2d7ee --- /dev/null +++ b/evidence/lab2_track2_write_cost.json @@ -0,0 +1,20 @@ +[ + { + "variant": "no_index", + "insert_10000_s": 1.37, + "indexes_size": "5152 kB", + "total_size": "19 MB" + }, + { + "variant": "chat_only", + "insert_10000_s": 1.093, + "indexes_size": "6648 kB", + "total_size": "21 MB" + }, + { + "variant": "full", + "insert_10000_s": 0.921, + "indexes_size": "15 MB", + "total_size": "31 MB" + } +] diff --git a/student-tests/test_track2.py b/student-tests/test_track2.py new file mode 100644 index 00000000..cbbfeb3d --- /dev/null +++ b/student-tests/test_track2.py @@ -0,0 +1,409 @@ +#!/usr/bin/env python3 +"""Трек 2 (оценка 5). План запроса, стоимость записи, backup/restore, RPO. + +Шаги: + 1. Сидируем >=100 000 сообщений с неравномерными чатами (track2_seed.sql). + 2. Сравниваем 3 варианта индексов на большом и маленьком чате: + - baseline (нет индекса) -> Seq Scan + - (chat_id) -> Bitmap/Index + Sort + - (chat_id, created_at, id) -> Index Scan без Sort (текущий) + Снимаем: план (EXPLAIN ANALYZE BUFFERS), время, shared hit/read. + 3. Стоимость записи: INSERT 10 000 строк на каждом варианте -> время вставки, + pg_indexes_size, pg_total_relation_size. + 4. Backup/restore: pg_dump -Fc, restore в новый контейнер postgres:18-alpine, + сверка COUNT(*), COUNT(*) GROUP BY chat_id и 100 случайных сообщений. + 5. Границы потери данных: во время pg_dump через API пишем новые сообщения, + после restore проверяем, что их нет. + +Все psql-вызовы идут через docker run postgres:18-alpine --add-host=host.docker.internal:host-gateway. +""" +import json +import os +import random +import subprocess +import sys +import time +import uuid +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, Failure, arguments, check, run # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / "evidence" +EVIDENCE.mkdir(exist_ok=True) + +SEED_SQL = ROOT / "student-tests" / "track2_seed.sql" + + +# ---------- helpers ---------- +def _read_dsn() -> str: + dsn = os.environ.get("DATABASE_URL") + if not dsn: + env_file = ROOT / ".env" + if env_file.is_file(): + for line in env_file.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line.startswith("DATABASE_URL="): + dsn = line.split("=", 1)[1].strip() + break + check(dsn, "DATABASE_URL не найден ни в окружении, ни в .env") + return dsn.replace("@localhost:", "@host.docker.internal:").replace( + "@127.0.0.1:", "@host.docker.internal:") + + +def _psql(dsn: str, sql: str, extra=None): + cmd = [ + "docker", "run", "--rm", + "--add-host=host.docker.internal:host-gateway", + "postgres:18-alpine", + "psql", dsn, "-tA", "-v", "ON_ERROR_STOP=1", + ] + if extra: + cmd += extra + cmd += ["-c", sql] + return subprocess.run(cmd, capture_output=True, text=True, timeout=300) + + +def _psql_file(dsn: str, path: Path): + cmd = [ + "docker", "run", "--rm", "-i", + "--add-host=host.docker.internal:host-gateway", + "-v", f"{path}:/seed.sql:ro", + "postgres:18-alpine", + "psql", dsn, "-v", "ON_ERROR_STOP=1", "-f", "/seed.sql", + ] + return subprocess.run(cmd, capture_output=True, text=True, timeout=600) + + +def _scalar(dsn: str, sql: str): + r = _psql(dsn, sql) + check(r.returncode == 0, "psql failed:\n" + r.stdout + r.stderr) + return r.stdout.strip().splitlines()[-1] + + +# ---------- 1. seed ---------- +def step_seed(dsn: str): + print("=== 1. seed ===") + before = int(_scalar(dsn, "SELECT count(*) FROM messages;")) + print(" messages before:", before) + r = _psql_file(dsn, SEED_SQL) + check(r.returncode == 0, "seed failed:\n" + r.stdout + r.stderr) + after = int(_scalar(dsn, "SELECT count(*) FROM messages;")) + print(" messages after:", after) + check(after >= 100_000, "seed не довёл до 100 000 (получено %d)" % after) + dist = _psql(dsn, "SELECT chat_id, count(*) FROM messages GROUP BY chat_id ORDER BY count(*) DESC;") + print(" распределение по чатам:\n" + dist.stdout) + return after + + +# ---------- 2. explain ---------- +def _big_and_small(dsn): + big = _scalar(dsn, "SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) DESC LIMIT 1;") + small = _scalar(dsn, "SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) ASC LIMIT 1;") + return big, small + + +def _explain(dsn: str, chat_id: str, label: str): + # Берём средний created_at, чтобы keyset реально резал историю + mid = _scalar(dsn, + "SELECT created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET (SELECT count(*)/2 FROM messages WHERE chat_id = '%s') LIMIT 1;" % (chat_id, chat_id)) + sql = ( + "EXPLAIN (ANALYZE, BUFFERS) " + "SELECT id, chat_id, sender_id, text, created_at FROM messages " + "WHERE chat_id = '%s' AND (created_at, id) > ('%s', '00000000-0000-0000-0000-000000000000') " + "ORDER BY created_at, id LIMIT 51;" % (chat_id, mid) + ) + r = _psql(dsn, sql) + check(r.returncode == 0, "EXPLAIN failed:\n" + r.stdout + r.stderr) + out = r.stdout + print("\n--- EXPLAIN %s (chat=%s...) ---" % (label, chat_id[:8])) + for line in out.splitlines(): + print(" ", line) + return out + + +def _drop_indexes(dsn): + _psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_created;") + _psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;") + + +def step_explain(dsn: str): + print("=== 2. EXPLAIN 3 варианта ===") + big, small = _big_and_small(dsn) + report = {"big_chat": big, "small_chat": small, "variants": {}} + + # (1) без индекса + _drop_indexes(dsn) + _psql(dsn, "ANALYZE messages;") + report["variants"]["no_index"] = { + "big": _explain(dsn, big, "no_index/big"), + "small": _explain(dsn, small, "no_index/small"), + } + + # (2) только (chat_id) + _psql(dsn, "CREATE INDEX idx_messages_chat_only ON messages (chat_id);") + _psql(dsn, "ANALYZE messages;") + report["variants"]["chat_only"] = { + "big": _explain(dsn, big, "chat_only/big"), + "small": _explain(dsn, small, "chat_only/small"), + } + + # (3) полный рабочий + _psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;") + _psql(dsn, "CREATE INDEX idx_messages_chat_created ON messages (chat_id, created_at, id);") + _psql(dsn, "ANALYZE messages;") + report["variants"]["full"] = { + "big": _explain(dsn, big, "full/big"), + "small": _explain(dsn, small, "full/small"), + } + + (EVIDENCE / "lab2_track2_explain.md").write_text( + "# Track 2: EXPLAIN (ANALYZE, BUFFERS)\n\n" + "## big chat\n```\n%s\n```\n\n## small chat\n```\n%s\n```\n\n" + "## no_index\n```\n%s\n```\n\n" + "## chat_only\n```\n%s\n```\n\n" + "## full (chat_id, created_at, id)\n```\n%s\n```\n" + % (report["variants"]["no_index"]["big"], + report["variants"]["no_index"]["small"], + report["variants"]["chat_only"]["big"], + report["variants"]["chat_only"]["small"], + report["variants"]["full"]["big"]), + encoding="utf-8", + ) + return report + + +# ---------- 3. write cost ---------- +def _index_sizes(dsn: str): + out = _scalar(dsn, "SELECT pg_size_pretty(pg_indexes_size('messages')) || '|' || pg_size_pretty(pg_total_relation_size('messages'));") + idx, total = out.split("|") + return idx.strip(), total.strip() + + +def step_write_cost(dsn: str): + print("=== 3. Стоимость записи: 10 000 INSERT на каждом варианте ===") + seed_user = _scalar(dsn, "SELECT id FROM users LIMIT 1;") + seed_chat = _scalar(dsn, "SELECT id FROM chats WHERE title = 'seed-chat-4' LIMIT 1;") + + results = [] + for name, setup in ( + ("no_index", lambda: _drop_indexes(dsn)), + ("chat_only", lambda: (_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_created;"), + _psql(dsn, "CREATE INDEX IF NOT EXISTS idx_messages_chat_only ON messages (chat_id);"))), + ("full", lambda: (_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;"), + _psql(dsn, "CREATE INDEX IF NOT EXISTS idx_messages_chat_created ON messages (chat_id, created_at, id);"))), + ): + setup() + # мерить именно вставку 10k + t0 = time.monotonic() + r = _psql(dsn, + "INSERT INTO messages (id, chat_id, sender_id, text, created_at) " + "SELECT gen_random_uuid(), '%s', '%s', 'wc-%s-' || g, now() " + "FROM generate_series(1, 10000) AS g;" % (seed_chat, seed_user, name)) + dt = time.monotonic() - t0 + check(r.returncode == 0, "insert failed:\n" + r.stdout + r.stderr) + idx, total = _index_sizes(dsn) + results.append({"variant": name, "insert_10000_s": round(dt, 3), + "indexes_size": idx, "total_size": total}) + print(" %-10s insert=%.3fs idx=%s total=%s" % (name, dt, idx, total)) + + (EVIDENCE / "lab2_track2_write_cost.json").write_text( + json.dumps(results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return results + + +# ---------- 4. backup/restore ---------- +def step_backup_restore(dsn: str): + print("=== 4. backup -> restore -> сверка ===") + out_file = EVIDENCE / "lab2_track2_dump.dump" + if out_file.exists(): + out_file.unlink() + + # pg_dump через docker run (использует образ PG 18) + t0 = time.monotonic() + r = subprocess.run( + ["docker", "run", "--rm", + "--add-host=host.docker.internal:host-gateway", + "-v", f"{EVIDENCE}:/out", + "postgres:18-alpine", + "pg_dump", "-Fc", "-f", "/out/lab2_track2_dump.dump", dsn], + capture_output=True, text=True, timeout=600, + ) + dump_time = time.monotonic() - t0 + check(r.returncode == 0, "pg_dump failed:\n" + r.stdout + r.stderr) + size = out_file.stat().st_size + print(" dump size = %.1f MiB, time = %.2fs" % (size / 1024 / 1024, dump_time)) + + # новый контейнер как отдельное хранилище + restore_container = "mtusi-pg-restore" + subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True) + subprocess.run( + ["docker", "run", "-d", "--name", restore_container, + "-e", "POSTGRES_PASSWORD=restore", "-e", "POSTGRES_USER=postgres", + "-e", "POSTGRES_DB=high_load_messenger", + "-p", "127.0.0.1:55432:5432", "postgres:18-alpine"], + capture_output=True, text=True, + ) + # ждём готовности + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + c = subprocess.run(["docker", "exec", restore_container, "pg_isready", "-U", "postgres"], + capture_output=True, text=True) + if c.returncode == 0: + break + time.sleep(1) + else: + raise Failure("restore container не поднялся") + + restore_dsn = "postgresql://postgres:restore@127.0.0.1:55432/high_load_messenger" + # pg_restore -c (clean) на пустую БД + t0 = time.monotonic() + r = subprocess.run( + ["docker", "run", "--rm", + "--add-host=host.docker.internal:host-gateway", + "--network", "host", + "-v", f"{EVIDENCE}:/in", + "postgres:18-alpine", + "pg_restore", "-d", "postgresql://postgres:restore@127.0.0.1:55432/high_load_messenger", + "--no-owner", "--no-privileges", "/in/lab2_track2_dump.dump"], + capture_output=True, text=True, timeout=600, + ) + restore_time = time.monotonic() - t0 + print(" pg_restore exit=%d, time=%.2fs" % (r.returncode, restore_time)) + check(r.returncode == 0, "pg_restore failed:\n" + r.stdout + r.stderr) + + # Сверка: сравнение с исходной БД через docker exec psql внутри restore-контейнера + def q_restore(sql): + c = subprocess.run(["docker", "exec", restore_container, "psql", "-U", "postgres", + "-d", "high_load_messenger", "-tA", "-c", sql], + capture_output=True, text=True, timeout=120) + check(c.returncode == 0, "restore psql failed:\n" + c.stdout + c.stderr) + return c.stdout.strip().splitlines()[-1] + + orig_total = int(_scalar(dsn, "SELECT count(*) FROM messages;")) + rest_total = int(q_restore("SELECT count(*) FROM messages;")) + check(orig_total == rest_total, "COUNT(*) mismatch: %d vs %d" % (orig_total, rest_total)) + + orig_dist = _scalar(dsn, + "SELECT string_agg(chat_id::text || ':' || c, ',' ORDER BY chat_id) " + "FROM (SELECT chat_id, count(*) c FROM messages GROUP BY chat_id) t;") + rest_dist = q_restore( + "SELECT string_agg(chat_id::text || ':' || c, ',' ORDER BY chat_id) " + "FROM (SELECT chat_id, count(*) c FROM messages GROUP BY chat_id) t;") + check(orig_dist == rest_dist, "распределение по чатам не совпадает") + + # 100 случайных сообщений + ids_raw = _scalar(dsn, "SELECT string_agg(id::text, ',') FROM (SELECT id FROM messages ORDER BY random() LIMIT 100) t;") + ids = ids_raw.split(",") + check(len(ids) == 100, "не удалось выбрать 100 id") + ids_list = ",".join("'%s'" % i for i in ids) + orig_rows = _scalar(dsn, + "SELECT string_agg(id::text || '|' || text, E'\n' ORDER BY id) FROM messages WHERE id IN (%s);" % ids_list) + rest_rows = q_restore( + "SELECT string_agg(id::text || '|' || text, E'\n' ORDER BY id) FROM messages WHERE id IN (%s);" % ids_list) + check(orig_rows == rest_rows, "100 случайных сообщений различаются") + + print(" OK: COUNT, распределение по чатам и 100 случайных сообщений совпадают") + print(" RTO (pg_restore) = %.2fs, dump = %.1f MiB" % (restore_time, size / 1024 / 1024)) + + (EVIDENCE / "lab2_track2_restore.txt").write_text( + "Track 2 restore\n" + "dump_time_s=%.3f\ndump_size_bytes=%d\nrestore_time_s=%.3f\n" + "count_original=%d\ncount_restored=%d\n" + "dist_match=true\nrandom100_match=true\n" + % (dump_time, size, restore_time, orig_total, rest_total), + encoding="utf-8") + + # чистим restore-контейнер (volume анонимный) + subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True) + return {"dump_time_s": round(dump_time, 3), "restore_time_s": round(restore_time, 3), + "dump_bytes": size, "count": orig_total} + + +# ---------- 5. RPO ---------- +def step_rpo(dsn: str, base_url: str, ca_file): + print("=== 5. RPO: запись во время pg_dump ===") + c = Client(base_url, ca_file) + alice, bob, eve, chat = c.bootstrap() + + marker = "rpo-" + uuid.uuid4().hex + # Запускаем pg_dump + out_file = EVIDENCE / "lab2_track2_rpo.dump" + if out_file.exists(): + out_file.unlink() + + proc = subprocess.Popen( + ["docker", "run", "--rm", + "--add-host=host.docker.internal:host-gateway", + "-v", f"{EVIDENCE}:/out", + "postgres:18-alpine", + "pg_dump", "-Fc", "-f", "/out/lab2_track2_rpo.dump", dsn], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + time.sleep(1.0) # гарантируем, что снапшот дампа уже снят + written = c.message(alice, chat, marker)["json"] + code = proc.wait(timeout=300) + check(code == 0, "pg_dump для RPO упал") + + # restore во временный контейнер + restore_container = "mtusi-pg-rpo" + subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True) + subprocess.run( + ["docker", "run", "-d", "--name", restore_container, + "-e", "POSTGRES_PASSWORD=restore", "-e", "POSTGRES_USER=postgres", + "-e", "POSTGRES_DB=high_load_messenger", + "postgres:18-alpine"], + capture_output=True, text=True, + ) + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + c2 = subprocess.run(["docker", "exec", restore_container, "pg_isready", "-U", "postgres"], + capture_output=True, text=True) + if c2.returncode == 0: + break + time.sleep(1) + + r = subprocess.run( + ["docker", "run", "--rm", + "--add-host=host.docker.internal:host-gateway", + "--network", "host", + "-v", f"{EVIDENCE}:/in", + "postgres:18-alpine", + "pg_restore", "-d", "postgresql://postgres:restore@127.0.0.1:5432/high_load_messenger", + "--no-owner", "--no-privileges", "/in/lab2_track2_rpo.dump"], + capture_output=True, text=True, timeout=600, + ) + check(r.returncode == 0, "pg_restore для RPO упал:\n" + r.stdout + r.stderr) + + c3 = subprocess.run(["docker", "exec", restore_container, "psql", "-U", "postgres", + "-d", "high_load_messenger", "-tA", "-c", + "SELECT count(*) FROM messages WHERE id = '%s';" % written["id"]], + capture_output=True, text=True) + present = int(c3.stdout.strip().splitlines()[-1]) + check(present == 0, "сообщение, записанное во время дампа, попало в дамп (RPO не 0)") + print(" OK: сообщение %s записано после старта дампа и в restore отсутствует" % written["id"][:8]) + + (EVIDENCE / "lab2_track2_rpo.txt").write_text( + "Track 2 RPO\n" + "marker_message_id=%s\npresent_in_restore=%d\nexpected=0\n" + % (written["id"], present), + encoding="utf-8") + + subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True) + + +# ---------- main ---------- +def main(): + args = arguments("Track 2: plan, write cost, backup/restore, RPO") + dsn = _read_dsn() + step_seed(dsn) + step_explain(dsn) + step_write_cost(dsn) + step_backup_restore(dsn) + step_rpo(dsn, args.base_url, args.ca_file) + print("PASS: track 2 (explain + write cost + backup/restore + RPO)") + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/student-tests/track2_explain.sql b/student-tests/track2_explain.sql new file mode 100644 index 00000000..5e28ed8a --- /dev/null +++ b/student-tests/track2_explain.sql @@ -0,0 +1,10 @@ +-- Track 2: сравнение планов и стоимости записи. +-- Все три варианта прогоняются на одном большом чате (60k) и одном маленьком (~1400). + +-- 0. Определяем два чата +\set ON_ERROR_STOP on + +SELECT id AS big_chat_id +FROM messages GROUP BY id_chat GROUP BY chat_id ORDER BY count(*) DESC LIMIT 1; + +SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) ASC LIMIT 1; \ No newline at end of file diff --git a/student-tests/track2_seed.sql b/student-tests/track2_seed.sql new file mode 100644 index 00000000..14a430de --- /dev/null +++ b/student-tests/track2_seed.sql @@ -0,0 +1,79 @@ +-- Track 2: сидирование >=100 000 сообщений с неравномерными чатами. +-- Запускать через: docker run --rm ... psql "$DATABASE_URL" -f track2_seed.sql +-- Идемпотентно по количеству: если сообщений уже >= 100 000, ничего не делает. + +DO $$ +DECLARE + v_user UUID; + v_total INT; + v_have INT; + v_need INT; + v_chat UUID; + rec RECORD; +BEGIN + SELECT count(*) INTO v_have FROM messages; + v_total := 100000; + IF v_have >= v_total THEN + RAISE NOTICE 'messages already >= %, skip seed', v_total; + RETURN; + END IF; + v_need := v_total - v_have; + + -- Гарантируем хотя бы одного пользователя + SELECT id INTO v_user FROM users LIMIT 1; + IF v_user IS NULL THEN + v_user := gen_random_uuid(); + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (v_user, 'seed_' || substr(v_user::text, 1, 8), 'Seed', + 'scrypt$16384$8$1$00$00', now()); + END IF; + + -- Создаём 10 чатов, если их меньше (нужны только для распределения) + -- Неравномерность: 60k / 20k / 10k / по ~1400 на остальные 7. + -- Считаем веса, доливаем только v_need строк. + CREATE TEMP TABLE _plan(chat_no INT, weight INT) ON COMMIT DROP; + INSERT INTO _plan VALUES (1, 60), (2, 20), (3, 10), + (4, 2), (5, 2), (6, 1), (7, 1), + (8, 1), (9, 1), (10, 1); + + FOR rec IN SELECT * FROM _plan ORDER BY chat_no LOOP + -- Находим/создаём chat_id для этого номера: используем title 'seed-chat-' + SELECT c.id INTO v_chat FROM chats c WHERE c.title = 'seed-chat-' || rec.chat_no LIMIT 1; + IF v_chat IS NULL THEN + v_chat := gen_random_uuid(); + INSERT INTO chats (id, title, created_at) VALUES (v_chat, 'seed-chat-' || rec.chat_no, now()); + END IF; + IF NOT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = v_chat AND user_id = v_user) THEN + INSERT INTO chat_members (chat_id, user_id) VALUES (v_chat, v_user); + END IF; + END LOOP; + + -- Распределяем v_need по весам (целочисленно), последний чат добирает остаток. + DECLARE + v_sum INT; + v_acc INT := 0; + v_alloc INT; + BEGIN + SELECT sum(weight) INTO v_sum FROM _plan; + FOR rec IN SELECT * FROM _plan ORDER BY chat_no LOOP + IF rec.chat_no = 10 THEN + v_alloc := v_need - v_acc; -- остаток + ELSE + v_alloc := (v_need::numeric * rec.weight / v_sum)::int; + v_acc := v_acc + v_alloc; + END IF; + + SELECT c.id INTO v_chat FROM chats c WHERE c.title = 'seed-chat-' || rec.chat_no LIMIT 1; + + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), v_chat, v_user, + 'seed-' || rec.chat_no || '-' || g, + now() - (g || ' milliseconds')::interval + FROM generate_series(1, v_alloc) AS g; + END LOOP; + END; +END $$; + +ANALYZE messages; +ANALYZE chat_members; +ANALYZE chats; \ No newline at end of file