реализовала уровень 5, трек 2
This commit is contained in:
@@ -0,0 +1,104 @@
|
||||
# Track 2: EXPLAIN (ANALYZE, BUFFERS)
|
||||
|
||||
## big chat
|
||||
```
|
||||
Limit (cost=4726.04..4731.85 rows=51 width=69) (actual time=28.316..32.077 rows=51.00 loops=1)
|
||||
Buffers: shared hit=1639
|
||||
-> Gather Merge (cost=4726.04..9161.77 rows=38920 width=69) (actual time=28.313..32.066 rows=51.00 loops=1)
|
||||
Workers Planned: 1
|
||||
Workers Launched: 1
|
||||
Buffers: shared hit=1639
|
||||
-> Sort (cost=3726.03..3783.26 rows=22894 width=69) (actual time=6.294..6.298 rows=25.50 loops=2)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: top-N heapsort Memory: 38kB
|
||||
Buffers: shared hit=1639
|
||||
Worker 0: Sort Method: quicksort Memory: 25kB
|
||||
-> Parallel Seq Scan on messages (cost=0.00..2962.24 rows=22894 width=69) (actual time=0.010..4.738 rows=13636.50 loops=2)
|
||||
Filter: ((chat_id = '79d37065-bfb0-4cb1-8f1e-e79533f724ff'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 15:03:17.148852+03'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid)))
|
||||
Rows Removed by Filter: 51364
|
||||
Buffers: shared hit=1624
|
||||
Planning:
|
||||
Buffers: shared hit=88
|
||||
Planning Time: 1.487 ms
|
||||
Execution Time: 32.133 ms
|
||||
|
||||
```
|
||||
|
||||
## small chat
|
||||
```
|
||||
Limit (cost=3899.08..3899.09 rows=6 width=69) (actual time=7.785..7.787 rows=1.00 loops=1)
|
||||
Buffers: shared hit=1630
|
||||
-> Sort (cost=3899.08..3899.09 rows=6 width=69) (actual time=7.783..7.783 rows=1.00 loops=1)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: quicksort Memory: 25kB
|
||||
Buffers: shared hit=1630
|
||||
-> Seq Scan on messages (cost=0.00..3899.00 rows=6 width=69) (actual time=0.056..7.764 rows=1.00 loops=1)
|
||||
Filter: ((chat_id = '730e474b-405e-4558-8cf3-ac6a17c2a7d1'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 01:23:13.456347+03'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid)))
|
||||
Rows Removed by Filter: 129999
|
||||
Buffers: shared hit=1624
|
||||
Planning:
|
||||
Buffers: shared hit=88
|
||||
Planning Time: 1.182 ms
|
||||
Execution Time: 7.820 ms
|
||||
|
||||
```
|
||||
|
||||
## no_index
|
||||
```
|
||||
Limit (cost=4475.27..4475.40 rows=51 width=69) (actual time=12.589..12.596 rows=51.00 loops=1)
|
||||
Buffers: shared hit=729
|
||||
-> Sort (cost=4475.27..4572.12 rows=38739 width=69) (actual time=12.587..12.589 rows=51.00 loops=1)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: top-N heapsort Memory: 38kB
|
||||
Buffers: shared hit=729
|
||||
-> Bitmap Heap Scan on messages (cost=609.04..3182.85 rows=38739 width=69) (actual time=0.891..8.050 rows=27273.00 loops=1)
|
||||
Recheck Cond: (chat_id = '79d37065-bfb0-4cb1-8f1e-e79533f724ff'::uuid)
|
||||
Filter: (ROW(created_at, id) > ROW('2026-10-02 15:03:17.148852+03'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))
|
||||
Rows Removed by Filter: 27272
|
||||
Heap Blocks: exact=675
|
||||
Buffers: shared hit=723
|
||||
-> Bitmap Index Scan on idx_messages_chat_only (cost=0.00..599.36 rows=54275 width=0) (actual time=0.823..0.823 rows=54545.00 loops=1)
|
||||
Index Cond: (chat_id = '79d37065-bfb0-4cb1-8f1e-e79533f724ff'::uuid)
|
||||
Index Searches: 1
|
||||
Buffers: shared hit=48
|
||||
Planning:
|
||||
Buffers: shared hit=100
|
||||
Planning Time: 2.045 ms
|
||||
Execution Time: 12.641 ms
|
||||
|
||||
```
|
||||
|
||||
## chat_only
|
||||
```
|
||||
Limit (cost=23.89..23.91 rows=8 width=69) (actual time=0.037..0.037 rows=1.00 loops=1)
|
||||
Buffers: shared hit=9
|
||||
-> Sort (cost=23.89..23.91 rows=8 width=69) (actual time=0.035..0.035 rows=1.00 loops=1)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: quicksort Memory: 25kB
|
||||
Buffers: shared hit=9
|
||||
-> Index Scan using idx_messages_chat_only on messages (cost=0.29..23.77 rows=8 width=69) (actual time=0.015..0.015 rows=1.00 loops=1)
|
||||
Index Cond: (chat_id = '730e474b-405e-4558-8cf3-ac6a17c2a7d1'::uuid)
|
||||
Filter: (ROW(created_at, id) > ROW('2026-10-02 01:23:13.456347+03'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))
|
||||
Index Searches: 1
|
||||
Buffers: shared hit=3
|
||||
Planning:
|
||||
Buffers: shared hit=100
|
||||
Planning Time: 2.240 ms
|
||||
Execution Time: 0.066 ms
|
||||
|
||||
```
|
||||
|
||||
## full (chat_id, created_at, id)
|
||||
```
|
||||
Limit (cost=0.42..9.42 rows=51 width=69) (actual time=0.101..0.118 rows=51.00 loops=1)
|
||||
Buffers: shared hit=5 read=1
|
||||
-> Index Scan using idx_messages_chat_created on messages (cost=0.42..6871.59 rows=38939 width=69) (actual time=0.098..0.112 rows=51.00 loops=1)
|
||||
Index Cond: ((chat_id = '79d37065-bfb0-4cb1-8f1e-e79533f724ff'::uuid) AND (ROW(created_at, id) > ROW('2026-10-02 15:03:17.148852+03'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid)))
|
||||
Index Searches: 1
|
||||
Buffers: shared hit=5 read=1
|
||||
Planning:
|
||||
Buffers: shared hit=111
|
||||
Planning Time: 1.697 ms
|
||||
Execution Time: 0.153 ms
|
||||
|
||||
```
|
||||
@@ -0,0 +1,20 @@
|
||||
[
|
||||
{
|
||||
"variant": "no_index",
|
||||
"insert_10000_s": 1.37,
|
||||
"indexes_size": "5152 kB",
|
||||
"total_size": "19 MB"
|
||||
},
|
||||
{
|
||||
"variant": "chat_only",
|
||||
"insert_10000_s": 1.093,
|
||||
"indexes_size": "6648 kB",
|
||||
"total_size": "21 MB"
|
||||
},
|
||||
{
|
||||
"variant": "full",
|
||||
"insert_10000_s": 0.921,
|
||||
"indexes_size": "15 MB",
|
||||
"total_size": "31 MB"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,409 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Трек 2 (оценка 5). План запроса, стоимость записи, backup/restore, RPO.
|
||||
|
||||
Шаги:
|
||||
1. Сидируем >=100 000 сообщений с неравномерными чатами (track2_seed.sql).
|
||||
2. Сравниваем 3 варианта индексов на большом и маленьком чате:
|
||||
- baseline (нет индекса) -> Seq Scan
|
||||
- (chat_id) -> Bitmap/Index + Sort
|
||||
- (chat_id, created_at, id) -> Index Scan без Sort (текущий)
|
||||
Снимаем: план (EXPLAIN ANALYZE BUFFERS), время, shared hit/read.
|
||||
3. Стоимость записи: INSERT 10 000 строк на каждом варианте -> время вставки,
|
||||
pg_indexes_size, pg_total_relation_size.
|
||||
4. Backup/restore: pg_dump -Fc, restore в новый контейнер postgres:18-alpine,
|
||||
сверка COUNT(*), COUNT(*) GROUP BY chat_id и 100 случайных сообщений.
|
||||
5. Границы потери данных: во время pg_dump через API пишем новые сообщения,
|
||||
после restore проверяем, что их нет.
|
||||
|
||||
Все psql-вызовы идут через docker run postgres:18-alpine --add-host=host.docker.internal:host-gateway.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, Failure, arguments, check, run # noqa: E402
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = ROOT / "evidence"
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
|
||||
SEED_SQL = ROOT / "student-tests" / "track2_seed.sql"
|
||||
|
||||
|
||||
# ---------- helpers ----------
|
||||
def _read_dsn() -> str:
|
||||
dsn = os.environ.get("DATABASE_URL")
|
||||
if not dsn:
|
||||
env_file = ROOT / ".env"
|
||||
if env_file.is_file():
|
||||
for line in env_file.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("DATABASE_URL="):
|
||||
dsn = line.split("=", 1)[1].strip()
|
||||
break
|
||||
check(dsn, "DATABASE_URL не найден ни в окружении, ни в .env")
|
||||
return dsn.replace("@localhost:", "@host.docker.internal:").replace(
|
||||
"@127.0.0.1:", "@host.docker.internal:")
|
||||
|
||||
|
||||
def _psql(dsn: str, sql: str, extra=None):
|
||||
cmd = [
|
||||
"docker", "run", "--rm",
|
||||
"--add-host=host.docker.internal:host-gateway",
|
||||
"postgres:18-alpine",
|
||||
"psql", dsn, "-tA", "-v", "ON_ERROR_STOP=1",
|
||||
]
|
||||
if extra:
|
||||
cmd += extra
|
||||
cmd += ["-c", sql]
|
||||
return subprocess.run(cmd, capture_output=True, text=True, timeout=300)
|
||||
|
||||
|
||||
def _psql_file(dsn: str, path: Path):
|
||||
cmd = [
|
||||
"docker", "run", "--rm", "-i",
|
||||
"--add-host=host.docker.internal:host-gateway",
|
||||
"-v", f"{path}:/seed.sql:ro",
|
||||
"postgres:18-alpine",
|
||||
"psql", dsn, "-v", "ON_ERROR_STOP=1", "-f", "/seed.sql",
|
||||
]
|
||||
return subprocess.run(cmd, capture_output=True, text=True, timeout=600)
|
||||
|
||||
|
||||
def _scalar(dsn: str, sql: str):
|
||||
r = _psql(dsn, sql)
|
||||
check(r.returncode == 0, "psql failed:\n" + r.stdout + r.stderr)
|
||||
return r.stdout.strip().splitlines()[-1]
|
||||
|
||||
|
||||
# ---------- 1. seed ----------
|
||||
def step_seed(dsn: str):
|
||||
print("=== 1. seed ===")
|
||||
before = int(_scalar(dsn, "SELECT count(*) FROM messages;"))
|
||||
print(" messages before:", before)
|
||||
r = _psql_file(dsn, SEED_SQL)
|
||||
check(r.returncode == 0, "seed failed:\n" + r.stdout + r.stderr)
|
||||
after = int(_scalar(dsn, "SELECT count(*) FROM messages;"))
|
||||
print(" messages after:", after)
|
||||
check(after >= 100_000, "seed не довёл до 100 000 (получено %d)" % after)
|
||||
dist = _psql(dsn, "SELECT chat_id, count(*) FROM messages GROUP BY chat_id ORDER BY count(*) DESC;")
|
||||
print(" распределение по чатам:\n" + dist.stdout)
|
||||
return after
|
||||
|
||||
|
||||
# ---------- 2. explain ----------
|
||||
def _big_and_small(dsn):
|
||||
big = _scalar(dsn, "SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) DESC LIMIT 1;")
|
||||
small = _scalar(dsn, "SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) ASC LIMIT 1;")
|
||||
return big, small
|
||||
|
||||
|
||||
def _explain(dsn: str, chat_id: str, label: str):
|
||||
# Берём средний created_at, чтобы keyset реально резал историю
|
||||
mid = _scalar(dsn,
|
||||
"SELECT created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET (SELECT count(*)/2 FROM messages WHERE chat_id = '%s') LIMIT 1;" % (chat_id, chat_id))
|
||||
sql = (
|
||||
"EXPLAIN (ANALYZE, BUFFERS) "
|
||||
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
|
||||
"WHERE chat_id = '%s' AND (created_at, id) > ('%s', '00000000-0000-0000-0000-000000000000') "
|
||||
"ORDER BY created_at, id LIMIT 51;" % (chat_id, mid)
|
||||
)
|
||||
r = _psql(dsn, sql)
|
||||
check(r.returncode == 0, "EXPLAIN failed:\n" + r.stdout + r.stderr)
|
||||
out = r.stdout
|
||||
print("\n--- EXPLAIN %s (chat=%s...) ---" % (label, chat_id[:8]))
|
||||
for line in out.splitlines():
|
||||
print(" ", line)
|
||||
return out
|
||||
|
||||
|
||||
def _drop_indexes(dsn):
|
||||
_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_created;")
|
||||
_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;")
|
||||
|
||||
|
||||
def step_explain(dsn: str):
|
||||
print("=== 2. EXPLAIN 3 варианта ===")
|
||||
big, small = _big_and_small(dsn)
|
||||
report = {"big_chat": big, "small_chat": small, "variants": {}}
|
||||
|
||||
# (1) без индекса
|
||||
_drop_indexes(dsn)
|
||||
_psql(dsn, "ANALYZE messages;")
|
||||
report["variants"]["no_index"] = {
|
||||
"big": _explain(dsn, big, "no_index/big"),
|
||||
"small": _explain(dsn, small, "no_index/small"),
|
||||
}
|
||||
|
||||
# (2) только (chat_id)
|
||||
_psql(dsn, "CREATE INDEX idx_messages_chat_only ON messages (chat_id);")
|
||||
_psql(dsn, "ANALYZE messages;")
|
||||
report["variants"]["chat_only"] = {
|
||||
"big": _explain(dsn, big, "chat_only/big"),
|
||||
"small": _explain(dsn, small, "chat_only/small"),
|
||||
}
|
||||
|
||||
# (3) полный рабочий
|
||||
_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;")
|
||||
_psql(dsn, "CREATE INDEX idx_messages_chat_created ON messages (chat_id, created_at, id);")
|
||||
_psql(dsn, "ANALYZE messages;")
|
||||
report["variants"]["full"] = {
|
||||
"big": _explain(dsn, big, "full/big"),
|
||||
"small": _explain(dsn, small, "full/small"),
|
||||
}
|
||||
|
||||
(EVIDENCE / "lab2_track2_explain.md").write_text(
|
||||
"# Track 2: EXPLAIN (ANALYZE, BUFFERS)\n\n"
|
||||
"## big chat\n```\n%s\n```\n\n## small chat\n```\n%s\n```\n\n"
|
||||
"## no_index\n```\n%s\n```\n\n"
|
||||
"## chat_only\n```\n%s\n```\n\n"
|
||||
"## full (chat_id, created_at, id)\n```\n%s\n```\n"
|
||||
% (report["variants"]["no_index"]["big"],
|
||||
report["variants"]["no_index"]["small"],
|
||||
report["variants"]["chat_only"]["big"],
|
||||
report["variants"]["chat_only"]["small"],
|
||||
report["variants"]["full"]["big"]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
# ---------- 3. write cost ----------
|
||||
def _index_sizes(dsn: str):
|
||||
out = _scalar(dsn, "SELECT pg_size_pretty(pg_indexes_size('messages')) || '|' || pg_size_pretty(pg_total_relation_size('messages'));")
|
||||
idx, total = out.split("|")
|
||||
return idx.strip(), total.strip()
|
||||
|
||||
|
||||
def step_write_cost(dsn: str):
|
||||
print("=== 3. Стоимость записи: 10 000 INSERT на каждом варианте ===")
|
||||
seed_user = _scalar(dsn, "SELECT id FROM users LIMIT 1;")
|
||||
seed_chat = _scalar(dsn, "SELECT id FROM chats WHERE title = 'seed-chat-4' LIMIT 1;")
|
||||
|
||||
results = []
|
||||
for name, setup in (
|
||||
("no_index", lambda: _drop_indexes(dsn)),
|
||||
("chat_only", lambda: (_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_created;"),
|
||||
_psql(dsn, "CREATE INDEX IF NOT EXISTS idx_messages_chat_only ON messages (chat_id);"))),
|
||||
("full", lambda: (_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;"),
|
||||
_psql(dsn, "CREATE INDEX IF NOT EXISTS idx_messages_chat_created ON messages (chat_id, created_at, id);"))),
|
||||
):
|
||||
setup()
|
||||
# мерить именно вставку 10k
|
||||
t0 = time.monotonic()
|
||||
r = _psql(dsn,
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) "
|
||||
"SELECT gen_random_uuid(), '%s', '%s', 'wc-%s-' || g, now() "
|
||||
"FROM generate_series(1, 10000) AS g;" % (seed_chat, seed_user, name))
|
||||
dt = time.monotonic() - t0
|
||||
check(r.returncode == 0, "insert failed:\n" + r.stdout + r.stderr)
|
||||
idx, total = _index_sizes(dsn)
|
||||
results.append({"variant": name, "insert_10000_s": round(dt, 3),
|
||||
"indexes_size": idx, "total_size": total})
|
||||
print(" %-10s insert=%.3fs idx=%s total=%s" % (name, dt, idx, total))
|
||||
|
||||
(EVIDENCE / "lab2_track2_write_cost.json").write_text(
|
||||
json.dumps(results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return results
|
||||
|
||||
|
||||
# ---------- 4. backup/restore ----------
|
||||
def step_backup_restore(dsn: str):
|
||||
print("=== 4. backup -> restore -> сверка ===")
|
||||
out_file = EVIDENCE / "lab2_track2_dump.dump"
|
||||
if out_file.exists():
|
||||
out_file.unlink()
|
||||
|
||||
# pg_dump через docker run (использует образ PG 18)
|
||||
t0 = time.monotonic()
|
||||
r = subprocess.run(
|
||||
["docker", "run", "--rm",
|
||||
"--add-host=host.docker.internal:host-gateway",
|
||||
"-v", f"{EVIDENCE}:/out",
|
||||
"postgres:18-alpine",
|
||||
"pg_dump", "-Fc", "-f", "/out/lab2_track2_dump.dump", dsn],
|
||||
capture_output=True, text=True, timeout=600,
|
||||
)
|
||||
dump_time = time.monotonic() - t0
|
||||
check(r.returncode == 0, "pg_dump failed:\n" + r.stdout + r.stderr)
|
||||
size = out_file.stat().st_size
|
||||
print(" dump size = %.1f MiB, time = %.2fs" % (size / 1024 / 1024, dump_time))
|
||||
|
||||
# новый контейнер как отдельное хранилище
|
||||
restore_container = "mtusi-pg-restore"
|
||||
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
|
||||
subprocess.run(
|
||||
["docker", "run", "-d", "--name", restore_container,
|
||||
"-e", "POSTGRES_PASSWORD=restore", "-e", "POSTGRES_USER=postgres",
|
||||
"-e", "POSTGRES_DB=high_load_messenger",
|
||||
"-p", "127.0.0.1:55432:5432", "postgres:18-alpine"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
# ждём готовности
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
c = subprocess.run(["docker", "exec", restore_container, "pg_isready", "-U", "postgres"],
|
||||
capture_output=True, text=True)
|
||||
if c.returncode == 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise Failure("restore container не поднялся")
|
||||
|
||||
restore_dsn = "postgresql://postgres:restore@127.0.0.1:55432/high_load_messenger"
|
||||
# pg_restore -c (clean) на пустую БД
|
||||
t0 = time.monotonic()
|
||||
r = subprocess.run(
|
||||
["docker", "run", "--rm",
|
||||
"--add-host=host.docker.internal:host-gateway",
|
||||
"--network", "host",
|
||||
"-v", f"{EVIDENCE}:/in",
|
||||
"postgres:18-alpine",
|
||||
"pg_restore", "-d", "postgresql://postgres:restore@127.0.0.1:55432/high_load_messenger",
|
||||
"--no-owner", "--no-privileges", "/in/lab2_track2_dump.dump"],
|
||||
capture_output=True, text=True, timeout=600,
|
||||
)
|
||||
restore_time = time.monotonic() - t0
|
||||
print(" pg_restore exit=%d, time=%.2fs" % (r.returncode, restore_time))
|
||||
check(r.returncode == 0, "pg_restore failed:\n" + r.stdout + r.stderr)
|
||||
|
||||
# Сверка: сравнение с исходной БД через docker exec psql внутри restore-контейнера
|
||||
def q_restore(sql):
|
||||
c = subprocess.run(["docker", "exec", restore_container, "psql", "-U", "postgres",
|
||||
"-d", "high_load_messenger", "-tA", "-c", sql],
|
||||
capture_output=True, text=True, timeout=120)
|
||||
check(c.returncode == 0, "restore psql failed:\n" + c.stdout + c.stderr)
|
||||
return c.stdout.strip().splitlines()[-1]
|
||||
|
||||
orig_total = int(_scalar(dsn, "SELECT count(*) FROM messages;"))
|
||||
rest_total = int(q_restore("SELECT count(*) FROM messages;"))
|
||||
check(orig_total == rest_total, "COUNT(*) mismatch: %d vs %d" % (orig_total, rest_total))
|
||||
|
||||
orig_dist = _scalar(dsn,
|
||||
"SELECT string_agg(chat_id::text || ':' || c, ',' ORDER BY chat_id) "
|
||||
"FROM (SELECT chat_id, count(*) c FROM messages GROUP BY chat_id) t;")
|
||||
rest_dist = q_restore(
|
||||
"SELECT string_agg(chat_id::text || ':' || c, ',' ORDER BY chat_id) "
|
||||
"FROM (SELECT chat_id, count(*) c FROM messages GROUP BY chat_id) t;")
|
||||
check(orig_dist == rest_dist, "распределение по чатам не совпадает")
|
||||
|
||||
# 100 случайных сообщений
|
||||
ids_raw = _scalar(dsn, "SELECT string_agg(id::text, ',') FROM (SELECT id FROM messages ORDER BY random() LIMIT 100) t;")
|
||||
ids = ids_raw.split(",")
|
||||
check(len(ids) == 100, "не удалось выбрать 100 id")
|
||||
ids_list = ",".join("'%s'" % i for i in ids)
|
||||
orig_rows = _scalar(dsn,
|
||||
"SELECT string_agg(id::text || '|' || text, E'\n' ORDER BY id) FROM messages WHERE id IN (%s);" % ids_list)
|
||||
rest_rows = q_restore(
|
||||
"SELECT string_agg(id::text || '|' || text, E'\n' ORDER BY id) FROM messages WHERE id IN (%s);" % ids_list)
|
||||
check(orig_rows == rest_rows, "100 случайных сообщений различаются")
|
||||
|
||||
print(" OK: COUNT, распределение по чатам и 100 случайных сообщений совпадают")
|
||||
print(" RTO (pg_restore) = %.2fs, dump = %.1f MiB" % (restore_time, size / 1024 / 1024))
|
||||
|
||||
(EVIDENCE / "lab2_track2_restore.txt").write_text(
|
||||
"Track 2 restore\n"
|
||||
"dump_time_s=%.3f\ndump_size_bytes=%d\nrestore_time_s=%.3f\n"
|
||||
"count_original=%d\ncount_restored=%d\n"
|
||||
"dist_match=true\nrandom100_match=true\n"
|
||||
% (dump_time, size, restore_time, orig_total, rest_total),
|
||||
encoding="utf-8")
|
||||
|
||||
# чистим restore-контейнер (volume анонимный)
|
||||
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
|
||||
return {"dump_time_s": round(dump_time, 3), "restore_time_s": round(restore_time, 3),
|
||||
"dump_bytes": size, "count": orig_total}
|
||||
|
||||
|
||||
# ---------- 5. RPO ----------
|
||||
def step_rpo(dsn: str, base_url: str, ca_file):
|
||||
print("=== 5. RPO: запись во время pg_dump ===")
|
||||
c = Client(base_url, ca_file)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
|
||||
marker = "rpo-" + uuid.uuid4().hex
|
||||
# Запускаем pg_dump
|
||||
out_file = EVIDENCE / "lab2_track2_rpo.dump"
|
||||
if out_file.exists():
|
||||
out_file.unlink()
|
||||
|
||||
proc = subprocess.Popen(
|
||||
["docker", "run", "--rm",
|
||||
"--add-host=host.docker.internal:host-gateway",
|
||||
"-v", f"{EVIDENCE}:/out",
|
||||
"postgres:18-alpine",
|
||||
"pg_dump", "-Fc", "-f", "/out/lab2_track2_rpo.dump", dsn],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
)
|
||||
time.sleep(1.0) # гарантируем, что снапшот дампа уже снят
|
||||
written = c.message(alice, chat, marker)["json"]
|
||||
code = proc.wait(timeout=300)
|
||||
check(code == 0, "pg_dump для RPO упал")
|
||||
|
||||
# restore во временный контейнер
|
||||
restore_container = "mtusi-pg-rpo"
|
||||
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
|
||||
subprocess.run(
|
||||
["docker", "run", "-d", "--name", restore_container,
|
||||
"-e", "POSTGRES_PASSWORD=restore", "-e", "POSTGRES_USER=postgres",
|
||||
"-e", "POSTGRES_DB=high_load_messenger",
|
||||
"postgres:18-alpine"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
c2 = subprocess.run(["docker", "exec", restore_container, "pg_isready", "-U", "postgres"],
|
||||
capture_output=True, text=True)
|
||||
if c2.returncode == 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
|
||||
r = subprocess.run(
|
||||
["docker", "run", "--rm",
|
||||
"--add-host=host.docker.internal:host-gateway",
|
||||
"--network", "host",
|
||||
"-v", f"{EVIDENCE}:/in",
|
||||
"postgres:18-alpine",
|
||||
"pg_restore", "-d", "postgresql://postgres:restore@127.0.0.1:5432/high_load_messenger",
|
||||
"--no-owner", "--no-privileges", "/in/lab2_track2_rpo.dump"],
|
||||
capture_output=True, text=True, timeout=600,
|
||||
)
|
||||
check(r.returncode == 0, "pg_restore для RPO упал:\n" + r.stdout + r.stderr)
|
||||
|
||||
c3 = subprocess.run(["docker", "exec", restore_container, "psql", "-U", "postgres",
|
||||
"-d", "high_load_messenger", "-tA", "-c",
|
||||
"SELECT count(*) FROM messages WHERE id = '%s';" % written["id"]],
|
||||
capture_output=True, text=True)
|
||||
present = int(c3.stdout.strip().splitlines()[-1])
|
||||
check(present == 0, "сообщение, записанное во время дампа, попало в дамп (RPO не 0)")
|
||||
print(" OK: сообщение %s записано после старта дампа и в restore отсутствует" % written["id"][:8])
|
||||
|
||||
(EVIDENCE / "lab2_track2_rpo.txt").write_text(
|
||||
"Track 2 RPO\n"
|
||||
"marker_message_id=%s\npresent_in_restore=%d\nexpected=0\n"
|
||||
% (written["id"], present),
|
||||
encoding="utf-8")
|
||||
|
||||
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
|
||||
|
||||
|
||||
# ---------- main ----------
|
||||
def main():
|
||||
args = arguments("Track 2: plan, write cost, backup/restore, RPO")
|
||||
dsn = _read_dsn()
|
||||
step_seed(dsn)
|
||||
step_explain(dsn)
|
||||
step_write_cost(dsn)
|
||||
step_backup_restore(dsn)
|
||||
step_rpo(dsn, args.base_url, args.ca_file)
|
||||
print("PASS: track 2 (explain + write cost + backup/restore + RPO)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,10 @@
|
||||
-- Track 2: сравнение планов и стоимости записи.
|
||||
-- Все три варианта прогоняются на одном большом чате (60k) и одном маленьком (~1400).
|
||||
|
||||
-- 0. Определяем два чата
|
||||
\set ON_ERROR_STOP on
|
||||
|
||||
SELECT id AS big_chat_id
|
||||
FROM messages GROUP BY id_chat GROUP BY chat_id ORDER BY count(*) DESC LIMIT 1;
|
||||
|
||||
SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) ASC LIMIT 1;
|
||||
@@ -0,0 +1,79 @@
|
||||
-- Track 2: сидирование >=100 000 сообщений с неравномерными чатами.
|
||||
-- Запускать через: docker run --rm ... psql "$DATABASE_URL" -f track2_seed.sql
|
||||
-- Идемпотентно по количеству: если сообщений уже >= 100 000, ничего не делает.
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
v_user UUID;
|
||||
v_total INT;
|
||||
v_have INT;
|
||||
v_need INT;
|
||||
v_chat UUID;
|
||||
rec RECORD;
|
||||
BEGIN
|
||||
SELECT count(*) INTO v_have FROM messages;
|
||||
v_total := 100000;
|
||||
IF v_have >= v_total THEN
|
||||
RAISE NOTICE 'messages already >= %, skip seed', v_total;
|
||||
RETURN;
|
||||
END IF;
|
||||
v_need := v_total - v_have;
|
||||
|
||||
-- Гарантируем хотя бы одного пользователя
|
||||
SELECT id INTO v_user FROM users LIMIT 1;
|
||||
IF v_user IS NULL THEN
|
||||
v_user := gen_random_uuid();
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES (v_user, 'seed_' || substr(v_user::text, 1, 8), 'Seed',
|
||||
'scrypt$16384$8$1$00$00', now());
|
||||
END IF;
|
||||
|
||||
-- Создаём 10 чатов, если их меньше (нужны только для распределения)
|
||||
-- Неравномерность: 60k / 20k / 10k / по ~1400 на остальные 7.
|
||||
-- Считаем веса, доливаем только v_need строк.
|
||||
CREATE TEMP TABLE _plan(chat_no INT, weight INT) ON COMMIT DROP;
|
||||
INSERT INTO _plan VALUES (1, 60), (2, 20), (3, 10),
|
||||
(4, 2), (5, 2), (6, 1), (7, 1),
|
||||
(8, 1), (9, 1), (10, 1);
|
||||
|
||||
FOR rec IN SELECT * FROM _plan ORDER BY chat_no LOOP
|
||||
-- Находим/создаём chat_id для этого номера: используем title 'seed-chat-<no>'
|
||||
SELECT c.id INTO v_chat FROM chats c WHERE c.title = 'seed-chat-' || rec.chat_no LIMIT 1;
|
||||
IF v_chat IS NULL THEN
|
||||
v_chat := gen_random_uuid();
|
||||
INSERT INTO chats (id, title, created_at) VALUES (v_chat, 'seed-chat-' || rec.chat_no, now());
|
||||
END IF;
|
||||
IF NOT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = v_chat AND user_id = v_user) THEN
|
||||
INSERT INTO chat_members (chat_id, user_id) VALUES (v_chat, v_user);
|
||||
END IF;
|
||||
END LOOP;
|
||||
|
||||
-- Распределяем v_need по весам (целочисленно), последний чат добирает остаток.
|
||||
DECLARE
|
||||
v_sum INT;
|
||||
v_acc INT := 0;
|
||||
v_alloc INT;
|
||||
BEGIN
|
||||
SELECT sum(weight) INTO v_sum FROM _plan;
|
||||
FOR rec IN SELECT * FROM _plan ORDER BY chat_no LOOP
|
||||
IF rec.chat_no = 10 THEN
|
||||
v_alloc := v_need - v_acc; -- остаток
|
||||
ELSE
|
||||
v_alloc := (v_need::numeric * rec.weight / v_sum)::int;
|
||||
v_acc := v_acc + v_alloc;
|
||||
END IF;
|
||||
|
||||
SELECT c.id INTO v_chat FROM chats c WHERE c.title = 'seed-chat-' || rec.chat_no LIMIT 1;
|
||||
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), v_chat, v_user,
|
||||
'seed-' || rec.chat_no || '-' || g,
|
||||
now() - (g || ' milliseconds')::interval
|
||||
FROM generate_series(1, v_alloc) AS g;
|
||||
END LOOP;
|
||||
END;
|
||||
END $$;
|
||||
|
||||
ANALYZE messages;
|
||||
ANALYZE chat_members;
|
||||
ANALYZE chats;
|
||||
Reference in New Issue
Block a user