439 lines
19 KiB
Python
439 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Трек 2 (оценка 5). План запроса, стоимость записи, backup/restore, RPO.
|
|
|
|
Шаги:
|
|
1. Сидируем >=100 000 сообщений с неравномерными чатами (track2_seed.sql).
|
|
2. Сравниваем 3 варианта индексов на большом и маленьком чате:
|
|
- baseline (нет индекса) -> Seq Scan
|
|
- (chat_id) -> Bitmap/Index + Sort
|
|
- (chat_id, created_at, id) -> Index Scan без Sort (текущий)
|
|
Снимаем: план (EXPLAIN ANALYZE BUFFERS), время, shared hit/read.
|
|
3. Стоимость записи: INSERT 10 000 строк на каждом варианте -> время вставки,
|
|
pg_indexes_size, pg_total_relation_size.
|
|
4. Backup/restore: pg_dump -Fc, restore в новый контейнер postgres:18-alpine,
|
|
сверка COUNT(*), COUNT(*) GROUP BY chat_id и 100 случайных сообщений.
|
|
5. Границы потери данных: во время pg_dump через API пишем новые сообщения,
|
|
после restore проверяем, что их нет.
|
|
|
|
Все psql-вызовы идут через docker run postgres:18-alpine в compose-сети проекта
|
|
(--network mtuci2026lab2_default), DSN указывает на хост postgres.
|
|
"""
|
|
import json
|
|
import os
|
|
import random
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
import base64
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
|
from client import Client, Failure, arguments, check, run # noqa: E402
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
EVIDENCE = ROOT / "evidence"
|
|
EVIDENCE.mkdir(exist_ok=True)
|
|
|
|
SEED_SQL = ROOT / "student-tests" / "track2_seed.sql"
|
|
|
|
# Имя compose-сети проекта: mtuci2026lab2_default (зафиксировано в compose.yaml networks.default.name).
|
|
COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab2_default")
|
|
|
|
|
|
# ---------- helpers ----------
|
|
def _read_dsn() -> str:
|
|
dsn = os.environ.get("DATABASE_URL")
|
|
if not dsn:
|
|
env_file = ROOT / ".env"
|
|
if env_file.is_file():
|
|
for line in env_file.read_text(encoding="utf-8").splitlines():
|
|
line = line.strip()
|
|
if line.startswith("DATABASE_URL="):
|
|
dsn = line.split("=", 1)[1].strip()
|
|
break
|
|
check(dsn, "DATABASE_URL не найден ни в окружении, ни в .env")
|
|
# Postgres доступен только внутри compose-сети; одноразовые psql/pg_dump
|
|
# контейнеры запускаются в этой же сети, имя хоста postgres резолвится как есть.
|
|
return dsn
|
|
|
|
|
|
def _psql(dsn: str, sql: str, extra=None):
|
|
cmd = [
|
|
"docker", "run", "--rm",
|
|
"--network", COMPOSE_NETWORK,
|
|
"postgres:18-alpine",
|
|
"psql", dsn, "-tA", "-v", "ON_ERROR_STOP=1",
|
|
]
|
|
if extra:
|
|
cmd += extra
|
|
cmd += ["-c", sql]
|
|
return subprocess.run(cmd, capture_output=True, text=True, timeout=300)
|
|
|
|
|
|
def _psql_file(dsn: str, path: Path):
|
|
cmd = [
|
|
"docker", "run", "--rm", "-i",
|
|
"--network", COMPOSE_NETWORK,
|
|
"-v", f"{path}:/seed.sql:ro",
|
|
"postgres:18-alpine",
|
|
"psql", dsn, "-v", "ON_ERROR_STOP=1", "-f", "/seed.sql",
|
|
]
|
|
return subprocess.run(cmd, capture_output=True, text=True, timeout=600)
|
|
|
|
|
|
def _scalar(dsn: str, sql: str):
|
|
r = _psql(dsn, sql)
|
|
check(r.returncode == 0, "psql failed:\n" + r.stdout + r.stderr)
|
|
return r.stdout.strip().splitlines()[-1]
|
|
|
|
|
|
# ---------- 1. seed ----------
|
|
def step_seed(dsn: str):
|
|
print("=== 1. seed ===")
|
|
before = int(_scalar(dsn, "SELECT count(*) FROM messages;"))
|
|
print(" messages before:", before)
|
|
r = _psql_file(dsn, SEED_SQL)
|
|
check(r.returncode == 0, "seed failed:\n" + r.stdout + r.stderr)
|
|
after = int(_scalar(dsn, "SELECT count(*) FROM messages;"))
|
|
print(" messages after:", after)
|
|
check(after >= 100_000, "seed не довёл до 100 000 (получено %d)" % after)
|
|
dist = _psql(dsn, "SELECT chat_id, count(*) FROM messages GROUP BY chat_id ORDER BY count(*) DESC;")
|
|
print(" распределение по чатам:\n" + dist.stdout)
|
|
return after
|
|
|
|
|
|
# ---------- 2. explain ----------
|
|
def _big_and_small(dsn):
|
|
big = _scalar(dsn, "SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) DESC LIMIT 1;")
|
|
small = _scalar(dsn, "SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) ASC LIMIT 1;")
|
|
return big, small
|
|
|
|
|
|
def _explain(dsn: str, chat_id: str, label: str):
|
|
# Берём средний created_at, чтобы keyset реально резал историю
|
|
mid = _scalar(dsn,
|
|
"SELECT created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET (SELECT count(*)/2 FROM messages WHERE chat_id = '%s') LIMIT 1;" % (chat_id, chat_id))
|
|
sql = (
|
|
"EXPLAIN (ANALYZE, BUFFERS) "
|
|
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
|
|
"WHERE chat_id = '%s' AND (created_at, id) > ('%s', '00000000-0000-0000-0000-000000000000') "
|
|
"ORDER BY created_at, id LIMIT 51;" % (chat_id, mid)
|
|
)
|
|
r = _psql(dsn, sql)
|
|
check(r.returncode == 0, "EXPLAIN failed:\n" + r.stdout + r.stderr)
|
|
out = r.stdout
|
|
print("\n--- EXPLAIN %s (chat=%s...) ---" % (label, chat_id[:8]))
|
|
for line in out.splitlines():
|
|
print(" ", line)
|
|
return out
|
|
|
|
|
|
def _drop_indexes(dsn):
|
|
_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_created;")
|
|
_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;")
|
|
|
|
|
|
def step_explain(dsn: str):
|
|
print("=== 2. EXPLAIN 3 варианта ===")
|
|
big, small = _big_and_small(dsn)
|
|
report = {"big_chat": big, "small_chat": small, "variants": {}}
|
|
|
|
# (1) без индекса
|
|
_drop_indexes(dsn)
|
|
_psql(dsn, "ANALYZE messages;")
|
|
report["variants"]["no_index"] = {
|
|
"big": _explain(dsn, big, "no_index/big"),
|
|
"small": _explain(dsn, small, "no_index/small"),
|
|
}
|
|
|
|
# (2) только (chat_id)
|
|
_psql(dsn, "CREATE INDEX idx_messages_chat_only ON messages (chat_id);")
|
|
_psql(dsn, "ANALYZE messages;")
|
|
report["variants"]["chat_only"] = {
|
|
"big": _explain(dsn, big, "chat_only/big"),
|
|
"small": _explain(dsn, small, "chat_only/small"),
|
|
}
|
|
|
|
# (3) полный рабочий
|
|
_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;")
|
|
_psql(dsn, "CREATE INDEX idx_messages_chat_created ON messages (chat_id, created_at, id);")
|
|
_psql(dsn, "ANALYZE messages;")
|
|
report["variants"]["full"] = {
|
|
"big": _explain(dsn, big, "full/big"),
|
|
"small": _explain(dsn, small, "full/small"),
|
|
}
|
|
|
|
(EVIDENCE / "lab2_track2_explain.md").write_text(
|
|
"# Track 2: EXPLAIN (ANALYZE, BUFFERS)\n\n"
|
|
"## big chat\n```\n%s\n```\n\n## small chat\n```\n%s\n```\n\n"
|
|
"## no_index\n```\n%s\n```\n\n"
|
|
"## chat_only\n```\n%s\n```\n\n"
|
|
"## full (chat_id, created_at, id)\n```\n%s\n```\n"
|
|
% (report["variants"]["no_index"]["big"],
|
|
report["variants"]["no_index"]["small"],
|
|
report["variants"]["chat_only"]["big"],
|
|
report["variants"]["chat_only"]["small"],
|
|
report["variants"]["full"]["big"]),
|
|
encoding="utf-8",
|
|
)
|
|
return report
|
|
|
|
|
|
# ---------- 3. write cost ----------
|
|
def _index_sizes(dsn: str):
|
|
out = _scalar(dsn, "SELECT pg_size_pretty(pg_indexes_size('messages')) || '|' || pg_size_pretty(pg_total_relation_size('messages'));")
|
|
idx, total = out.split("|")
|
|
return idx.strip(), total.strip()
|
|
|
|
|
|
def step_write_cost(dsn: str):
|
|
print("=== 3. Стоимость записи: 10 000 INSERT на каждом варианте ===")
|
|
seed_user = _scalar(dsn, "SELECT id FROM users LIMIT 1;")
|
|
seed_chat = _scalar(dsn, "SELECT id FROM chats WHERE title = 'seed-chat-4' LIMIT 1;")
|
|
|
|
results = []
|
|
for name, setup in (
|
|
("no_index", lambda: _drop_indexes(dsn)),
|
|
("chat_only", lambda: (_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_created;"),
|
|
_psql(dsn, "CREATE INDEX IF NOT EXISTS idx_messages_chat_only ON messages (chat_id);"))),
|
|
("full", lambda: (_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;"),
|
|
_psql(dsn, "CREATE INDEX IF NOT EXISTS idx_messages_chat_created ON messages (chat_id, created_at, id);"))),
|
|
):
|
|
setup()
|
|
# мерить именно вставку 10k
|
|
t0 = time.monotonic()
|
|
r = _psql(dsn,
|
|
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) "
|
|
"SELECT gen_random_uuid(), '%s', '%s', 'wc-%s-' || g, now() "
|
|
"FROM generate_series(1, 10000) AS g;" % (seed_chat, seed_user, name))
|
|
dt = time.monotonic() - t0
|
|
check(r.returncode == 0, "insert failed:\n" + r.stdout + r.stderr)
|
|
idx, total = _index_sizes(dsn)
|
|
results.append({"variant": name, "insert_10000_s": round(dt, 3),
|
|
"indexes_size": idx, "total_size": total})
|
|
print(" %-10s insert=%.3fs idx=%s total=%s" % (name, dt, idx, total))
|
|
|
|
(EVIDENCE / "lab2_track2_write_cost.json").write_text(
|
|
json.dumps(results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
return results
|
|
|
|
|
|
# ---------- 4. backup/restore ----------
|
|
def step_backup_restore(dsn: str):
|
|
print("=== 4. backup -> restore -> сверка ===")
|
|
out_file = EVIDENCE / "lab2_track2_dump.dump"
|
|
if out_file.exists():
|
|
out_file.unlink()
|
|
|
|
# pg_dump через docker run (использует образ PG 18), в compose-сети
|
|
t0 = time.monotonic()
|
|
r = subprocess.run(
|
|
["docker", "run", "--rm",
|
|
"--network", COMPOSE_NETWORK,
|
|
"-v", f"{EVIDENCE}:/out",
|
|
"postgres:18-alpine",
|
|
"pg_dump", "-Fc", "-f", "/out/lab2_track2_dump.dump", dsn],
|
|
capture_output=True, text=True, timeout=600,
|
|
)
|
|
dump_time = time.monotonic() - t0
|
|
check(r.returncode == 0, "pg_dump failed:\n" + r.stdout + r.stderr)
|
|
size = out_file.stat().st_size
|
|
print(" dump size = %.1f MiB, time = %.2fs" % (size / 1024 / 1024, dump_time))
|
|
|
|
# новый контейнер как отдельное хранилище — в той же compose-сети,
|
|
# порт наружу не публикуем
|
|
restore_container = "mtuci-pg-restore"
|
|
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
|
|
subprocess.run(
|
|
["docker", "run", "-d", "--name", restore_container,
|
|
"--network", COMPOSE_NETWORK,
|
|
"-e", "POSTGRES_PASSWORD=restore", "-e", "POSTGRES_USER=postgres",
|
|
"-e", "POSTGRES_DB=high_load_messenger",
|
|
"postgres:18-alpine"],
|
|
capture_output=True, text=True,
|
|
)
|
|
# ждём готовности
|
|
deadline = time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
c = subprocess.run(["docker", "exec", restore_container, "pg_isready", "-U", "postgres"],
|
|
capture_output=True, text=True)
|
|
if c.returncode == 0:
|
|
break
|
|
time.sleep(1)
|
|
else:
|
|
raise Failure("restore container не поднялся")
|
|
|
|
# pg_restore -c (clean) на пустую БД; restore-контейнер доступен по имени
|
|
# в compose-сети
|
|
t0 = time.monotonic()
|
|
r = subprocess.run(
|
|
["docker", "run", "--rm",
|
|
"--network", COMPOSE_NETWORK,
|
|
"-v", f"{EVIDENCE}:/in",
|
|
"postgres:18-alpine",
|
|
"pg_restore", "-d", "postgresql://postgres:restore@mtuci-pg-restore:5432/high_load_messenger",
|
|
"--no-owner", "--no-privileges", "/in/lab2_track2_dump.dump"],
|
|
capture_output=True, text=True, timeout=600,
|
|
)
|
|
restore_time = time.monotonic() - t0
|
|
print(" pg_restore exit=%d, time=%.2fs" % (r.returncode, restore_time))
|
|
check(r.returncode == 0, "pg_restore failed:\n" + r.stdout + r.stderr)
|
|
|
|
# Сверка: сравнение с исходной БД через docker exec psql внутри restore-контейнера
|
|
def q_restore(sql):
|
|
c = subprocess.run(["docker", "exec", restore_container, "psql", "-U", "postgres",
|
|
"-d", "high_load_messenger", "-tA", "-c", sql],
|
|
capture_output=True, text=True, timeout=120)
|
|
check(c.returncode == 0, "restore psql failed:\n" + c.stdout + c.stderr)
|
|
return c.stdout.strip().splitlines()[-1]
|
|
|
|
orig_total = int(_scalar(dsn, "SELECT count(*) FROM messages;"))
|
|
rest_total = int(q_restore("SELECT count(*) FROM messages;"))
|
|
check(orig_total == rest_total, "COUNT(*) mismatch: %d vs %d" % (orig_total, rest_total))
|
|
|
|
orig_dist = _scalar(dsn,
|
|
"SELECT string_agg(chat_id::text || ':' || c, ',' ORDER BY chat_id) "
|
|
"FROM (SELECT chat_id, count(*) c FROM messages GROUP BY chat_id) t;")
|
|
rest_dist = q_restore(
|
|
"SELECT string_agg(chat_id::text || ':' || c, ',' ORDER BY chat_id) "
|
|
"FROM (SELECT chat_id, count(*) c FROM messages GROUP BY chat_id) t;")
|
|
check(orig_dist == rest_dist, "распределение по чатам не совпадает")
|
|
|
|
# 100 случайных сообщений
|
|
ids_raw = _scalar(dsn, "SELECT string_agg(id::text, ',') FROM (SELECT id FROM messages ORDER BY random() LIMIT 100) t;")
|
|
ids = ids_raw.split(",")
|
|
check(len(ids) == 100, "не удалось выбрать 100 id")
|
|
ids_list = ",".join("'%s'" % i for i in ids)
|
|
orig_rows = _scalar(dsn,
|
|
"SELECT string_agg(id::text || '|' || text, E'\n' ORDER BY id) FROM messages WHERE id IN (%s);" % ids_list)
|
|
rest_rows = q_restore(
|
|
"SELECT string_agg(id::text || '|' || text, E'\n' ORDER BY id) FROM messages WHERE id IN (%s);" % ids_list)
|
|
check(orig_rows == rest_rows, "100 случайных сообщений различаются")
|
|
|
|
print(" OK: COUNT, распределение по чатам и 100 случайных сообщений совпадают")
|
|
print(" RTO (pg_restore) = %.2fs, dump = %.1f MiB" % (restore_time, size / 1024 / 1024))
|
|
|
|
(EVIDENCE / "lab2_track2_restore.txt").write_text(
|
|
"Track 2 restore\n"
|
|
"dump_time_s=%.3f\ndump_size_bytes=%d\nrestore_time_s=%.3f\n"
|
|
"count_original=%d\ncount_restored=%d\n"
|
|
"dist_match=true\nrandom100_match=true\n"
|
|
% (dump_time, size, restore_time, orig_total, rest_total),
|
|
encoding="utf-8")
|
|
|
|
# чистим restore-контейнер (volume анонимный)
|
|
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
|
|
return {"dump_time_s": round(dump_time, 3), "restore_time_s": round(restore_time, 3),
|
|
"dump_bytes": size, "count": orig_total}
|
|
|
|
|
|
# ---------- 5. RPO ----------
|
|
def step_rpo(dsn: str, base_url: str, ca_file):
|
|
print("=== 5. RPO: запись во время pg_dump ===")
|
|
c = Client(base_url, ca_file)
|
|
|
|
# Данные и сессии готовим заранее, до дампа.
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
|
|
# Контроль: сессия должна быть валидна, иначе 401 — это не RPO.
|
|
me = c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
|
|
check(me["status"] == 200, "сессия должна быть валидна перед RPO-тестом")
|
|
|
|
marker = "rpo-" + uuid.uuid4().hex
|
|
# Запускаем pg_dump
|
|
out_file = EVIDENCE / "lab2_track2_rpo.dump"
|
|
if out_file.exists():
|
|
out_file.unlink()
|
|
|
|
# pg_dump: тот же приём, что в шаге 4, в compose-сети.
|
|
proc = subprocess.Popen(
|
|
["docker", "run", "--rm",
|
|
"--network", COMPOSE_NETWORK,
|
|
"-v", f"{EVIDENCE}:/out",
|
|
"postgres:18-alpine",
|
|
"pg_dump", "-Fc", "-f", "/out/lab2_track2_rpo.dump", dsn],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
)
|
|
time.sleep(1.0) # снапшот уже снят
|
|
|
|
# Свежий логин прямо перед записью: не зависим от TTL старой сессии.
|
|
cred = base64.b64encode(
|
|
(alice["registration"]["username"] + ":" +
|
|
alice["registration"]["password"]).encode()
|
|
).decode()
|
|
fresh = c.call("POST", "/api/v1/auth/sessions", 201,
|
|
headers={"Authorization": "Basic " + cred},
|
|
schema="Session")["json"]
|
|
alice_for_write = {"id": alice["id"], "user": alice["user"], "token": fresh["token"]}
|
|
|
|
written = c.message(alice_for_write, chat, marker)["json"]
|
|
code = proc.wait(timeout=300)
|
|
check(code == 0, "pg_dump для RPO упал")
|
|
|
|
# Временный restore-контейнер: в compose-сети, без публикации портов.
|
|
restore_container = "mtuci-pg-rpo"
|
|
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
|
|
subprocess.run(
|
|
["docker", "run", "-d", "--name", restore_container,
|
|
"--network", COMPOSE_NETWORK,
|
|
"-e", "POSTGRES_PASSWORD=restore", "-e", "POSTGRES_USER=postgres",
|
|
"-e", "POSTGRES_DB=high_load_messenger",
|
|
"postgres:18-alpine"],
|
|
capture_output=True, text=True,
|
|
)
|
|
|
|
deadline = time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
c2 = subprocess.run(["docker", "exec", restore_container, "pg_isready", "-U", "postgres"],
|
|
capture_output=True, text=True)
|
|
if c2.returncode == 0:
|
|
break
|
|
time.sleep(1)
|
|
else:
|
|
raise Failure("restore container для RPO не поднялся")
|
|
|
|
# pg_restore идёт в restore-контейнер по имени в compose-сети.
|
|
r = subprocess.run(
|
|
["docker", "run", "--rm",
|
|
"--network", COMPOSE_NETWORK,
|
|
"-v", f"{EVIDENCE}:/in",
|
|
"postgres:18-alpine",
|
|
"pg_restore", "-d",
|
|
"postgresql://postgres:restore@mtuci-pg-rpo:5432/high_load_messenger",
|
|
"--no-owner", "--no-privileges", "/in/lab2_track2_rpo.dump"],
|
|
capture_output=True, text=True, timeout=600,
|
|
)
|
|
check(r.returncode == 0, "pg_restore для RPO упал:\n" + r.stdout + r.stderr)
|
|
|
|
c3 = subprocess.run(["docker", "exec", restore_container, "psql", "-U", "postgres",
|
|
"-d", "high_load_messenger", "-tA", "-c",
|
|
"SELECT count(*) FROM messages WHERE id = '%s';" % written["id"]],
|
|
capture_output=True, text=True)
|
|
present = int(c3.stdout.strip().splitlines()[-1])
|
|
check(present == 0, "сообщение, записанное во время дампа, попало в дамп (RPO не 0)")
|
|
print(" OK: сообщение %s записано после старта дампа и в restore отсутствует" % written["id"][:8])
|
|
|
|
(EVIDENCE / "lab2_track2_rpo.txt").write_text(
|
|
"Track 2 RPO\n"
|
|
"marker_message_id=%s\npresent_in_restore=%d\nexpected=0\n"
|
|
% (written["id"], present),
|
|
encoding="utf-8")
|
|
|
|
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
|
|
|
|
|
|
# ---------- main ----------
|
|
def main():
|
|
args = arguments("Track 2: plan, write cost, backup/restore, RPO")
|
|
dsn = _read_dsn()
|
|
step_seed(dsn)
|
|
step_explain(dsn)
|
|
step_write_cost(dsn)
|
|
step_backup_restore(dsn)
|
|
step_rpo(dsn, args.base_url, args.ca_file)
|
|
print("PASS: track 2 (explain + write cost + backup/restore + RPO)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main) |