Files
mtusi2026lab6/student-tests/lab2_test_track2.py
T

465 lines
21 KiB
Python

#!/usr/bin/env python3
"""Трек 2 (оценка 5). План запроса, стоимость записи, backup/restore, RPO.
Шаги:
1. Сидируем >=100 000 сообщений с неравномерными чатами (track2_seed.sql).
2. Сравниваем 3 варианта индексов на большом и маленьком чате:
- baseline (нет индекса) -> Seq Scan
- (chat_id) -> Bitmap/Index + Sort
- (chat_id, created_at, id) -> Index Scan без Sort (текущий)
Снимаем: план (EXPLAIN ANALYZE BUFFERS), время, shared hit/read.
3. Стоимость записи: INSERT 10 000 строк на каждом варианте -> время вставки,
pg_indexes_size, pg_total_relation_size.
4. Backup/restore: pg_dump -Fc, restore в новый контейнер postgres:18-alpine,
сверка COUNT(*), COUNT(*) GROUP BY chat_id и 100 случайных сообщений.
5. Границы потери данных: во время pg_dump через API пишем новые сообщения,
после restore проверяем, что их нет.
Все psql-вызовы идут через docker run postgres:18-alpine в compose-сети проекта
(--network mtuci2026lab2_default), DSN указывает на хост postgres.
"""
import json
import os
import random
import subprocess
import sys
import time
import uuid
from pathlib import Path
import base64
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = ROOT / "evidence"
EVIDENCE.mkdir(exist_ok=True)
SEED_SQL = ROOT / "student-tests" / "track2_seed.sql"
# Имя compose-сети проекта: mtuci2026lab2_default (зафиксировано в compose.yaml networks.default.name).
COMPOSE_NETWORK = os.environ.get("COMPOSE_NETWORK", "mtuci2026lab2_default")
# ---------- helpers ----------
def _read_dsn() -> str:
dsn = os.environ.get("DATABASE_URL")
if not dsn:
env_file = ROOT / ".env"
if env_file.is_file():
for line in env_file.read_text(encoding="utf-8").splitlines():
line = line.strip()
if line.startswith("DATABASE_URL="):
dsn = line.split("=", 1)[1].strip()
break
check(dsn, "DATABASE_URL не найден ни в окружении, ни в .env")
# Postgres доступен только внутри compose-сети; одноразовые psql/pg_dump
# контейнеры запускаются в этой же сети, имя хоста postgres резолвится как есть.
return dsn
def _psql(dsn: str, sql: str, extra=None):
cmd = [
"docker", "run", "--rm",
"--network", COMPOSE_NETWORK,
"postgres:18-alpine",
"psql", dsn, "-tA", "-v", "ON_ERROR_STOP=1",
]
if extra:
cmd += extra
cmd += ["-c", sql]
return subprocess.run(cmd, capture_output=True, text=True, timeout=300)
def _psql_file(dsn: str, path: Path):
cmd = [
"docker", "run", "--rm", "-i",
"--network", COMPOSE_NETWORK,
"-v", f"{path}:/seed.sql:ro",
"postgres:18-alpine",
"psql", dsn, "-v", "ON_ERROR_STOP=1", "-f", "/seed.sql",
]
return subprocess.run(cmd, capture_output=True, text=True, timeout=600)
def _scalar(dsn: str, sql: str):
r = _psql(dsn, sql)
check(r.returncode == 0, "psql failed:\n" + r.stdout + r.stderr)
return r.stdout.strip().splitlines()[-1]
# ---------- 1. seed ----------
def step_seed(dsn: str):
print("=== 1. seed ===")
before = int(_scalar(dsn, "SELECT count(*) FROM messages;"))
print(" messages before:", before)
r = _psql_file(dsn, SEED_SQL)
check(r.returncode == 0, "seed failed:\n" + r.stdout + r.stderr)
after = int(_scalar(dsn, "SELECT count(*) FROM messages;"))
print(" messages after:", after)
check(after >= 100_000, "seed не довёл до 100 000 (получено %d)" % after)
dist = _psql(dsn, "SELECT chat_id, count(*) FROM messages GROUP BY chat_id ORDER BY count(*) DESC;")
print(" распределение по чатам:\n" + dist.stdout)
return after
# ---------- 2. explain ----------
def _big_and_small(dsn):
big = _scalar(dsn, "SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) DESC LIMIT 1;")
small = _scalar(dsn, "SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) ASC LIMIT 1;")
return big, small
def _chat_size(dsn: str, chat_id: str) -> int:
return int(_scalar(dsn, "SELECT count(*) FROM messages WHERE chat_id = '%s';" % chat_id))
def _explain(dsn: str, chat_id: str, label: str):
# Берём средний created_at, чтобы keyset реально резал историю
mid = _scalar(dsn,
"SELECT created_at FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET (SELECT count(*)/2 FROM messages WHERE chat_id = '%s') LIMIT 1;" % (chat_id, chat_id))
sql = (
"EXPLAIN (ANALYZE, BUFFERS) "
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
"WHERE chat_id = '%s' AND (created_at, id) > ('%s', '00000000-0000-0000-0000-000000000000') "
"ORDER BY created_at, id LIMIT 51;" % (chat_id, mid)
)
r = _psql(dsn, sql)
check(r.returncode == 0, "EXPLAIN failed:\n" + r.stdout + r.stderr)
out = r.stdout
print("\n--- EXPLAIN %s (chat=%s...) ---" % (label, chat_id[:8]))
for line in out.splitlines():
print(" ", line)
return out
def _drop_indexes(dsn):
_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_created;")
_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;")
def _format_section(title: str, chat_size: int, plan: str, note: str = "") -> str:
"""Собирает один раздел отчёта в формате образца."""
head = "## %s\n\n" % title
if note:
head += note.rstrip() + "\n\n"
return head + "```\n" + plan.rstrip() + "\n```\n"
def step_explain(dsn: str, total_messages: int):
print("=== 2. EXPLAIN 3 варианта ===")
big, small = _big_and_small(dsn)
big_size = _chat_size(dsn, big)
small_size = _chat_size(dsn, small)
# (1) без индекса
_drop_indexes(dsn)
_psql(dsn, "ANALYZE messages;")
no_index_big = _explain(dsn, big, "no_index/big")
no_index_small = _explain(dsn, small, "no_index/small")
# (2) только (chat_id)
_psql(dsn, "CREATE INDEX idx_messages_chat_only ON messages (chat_id);")
_psql(dsn, "ANALYZE messages;")
chat_only_big = _explain(dsn, big, "chat_only/big")
chat_only_small = _explain(dsn, small, "chat_only/small")
# (3) полный рабочий
_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;")
_psql(dsn, "CREATE INDEX idx_messages_chat_created ON messages (chat_id, created_at, id);")
_psql(dsn, "ANALYZE messages;")
full_big = _explain(dsn, big, "full/big")
full_small = _explain(dsn, small, "full/small")
# Заголовок отчёта — как в образце
parts = []
parts.append(
"# Трек 2. EXPLAIN (ANALYZE, BUFFERS): 3 варианта индекса\n\n"
"Данные: %d сообщений, неравномерные чаты. "
"Запрос: `SELECT ... FROM messages WHERE chat_id = <id> AND (created_at, id) > (<середина>, …) "
"ORDER BY created_at, id LIMIT 51`.\n"
"Большой чат ~%d, маленький чат ~%d.\n" % (total_messages, big_size, small_size)
)
parts.append(_format_section(
"1. Без индекса, большой чат", big_size, no_index_big,
"`Parallel Seq Scan` по всей таблице, затем `Sort`."))
parts.append(_format_section(
"2. Без индекса, маленький чат", small_size, no_index_small,
"Тот же `Parallel Seq Scan` по всей таблице."))
parts.append(_format_section(
"3. Индекс (chat_id), большой чат", big_size, chat_only_big,
"`Bitmap Index Scan` по `idx_messages_chat_only`, затем `Bitmap Heap Scan` и `Sort`."))
parts.append(_format_section(
"4. Индекс (chat_id), маленький чат", small_size, chat_only_small,
"`Index Scan using idx_messages_chat_only` + `Sort`."))
parts.append(_format_section(
"5. Полный индекс (chat_id, created_at, id), большой чат", big_size, full_big,
"`Index Scan using idx_messages_chat_created`, **без `Sort`**."))
parts.append(_format_section(
"6. Полный индекс (chat_id, created_at, id), маленький чат", small_size, full_small,
"`Index Scan` без `Sort`."))
(EVIDENCE / "lab2_track2_explain.md").write_text(
"\n".join(parts), encoding="utf-8")
return {"big_chat": big, "small_chat": small}
# ---------- 3. write cost ----------
def _index_sizes(dsn: str):
out = _scalar(dsn, "SELECT pg_size_pretty(pg_indexes_size('messages')) || '|' || pg_size_pretty(pg_total_relation_size('messages'));")
idx, total = out.split("|")
return idx.strip(), total.strip()
def step_write_cost(dsn: str):
print("=== 3. Стоимость записи: 10 000 INSERT на каждом варианте ===")
seed_user = _scalar(dsn, "SELECT id FROM users LIMIT 1;")
seed_chat = _scalar(dsn, "SELECT id FROM chats WHERE title = 'seed-chat-4' LIMIT 1;")
results = []
for name, setup in (
("no_index", lambda: _drop_indexes(dsn)),
("chat_only", lambda: (_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_created;"),
_psql(dsn, "CREATE INDEX IF NOT EXISTS idx_messages_chat_only ON messages (chat_id);"))),
("full", lambda: (_psql(dsn, "DROP INDEX IF EXISTS idx_messages_chat_only;"),
_psql(dsn, "CREATE INDEX IF NOT EXISTS idx_messages_chat_created ON messages (chat_id, created_at, id);"))),
):
setup()
# мерить именно вставку 10k
t0 = time.monotonic()
r = _psql(dsn,
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) "
"SELECT gen_random_uuid(), '%s', '%s', 'wc-%s-' || g, now() "
"FROM generate_series(1, 10000) AS g;" % (seed_chat, seed_user, name))
dt = time.monotonic() - t0
check(r.returncode == 0, "insert failed:\n" + r.stdout + r.stderr)
idx, total = _index_sizes(dsn)
results.append({"variant": name, "insert_10000_s": round(dt, 3),
"indexes_size": idx, "total_size": total})
print(" %-10s insert=%.3fs idx=%s total=%s" % (name, dt, idx, total))
(EVIDENCE / "lab2_track2_write_cost.json").write_text(
json.dumps(results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return results
# ---------- 4. backup/restore ----------
def step_backup_restore(dsn: str):
print("=== 4. backup -> restore -> сверка ===")
out_file = EVIDENCE / "lab2_track2_dump.dump"
if out_file.exists():
out_file.unlink()
# pg_dump через docker run (использует образ PG 18), в compose-сети
t0 = time.monotonic()
r = subprocess.run(
["docker", "run", "--rm",
"--network", COMPOSE_NETWORK,
"-v", f"{EVIDENCE}:/out",
"postgres:18-alpine",
"pg_dump", "-Fc", "-f", "/out/lab2_track2_dump.dump", dsn],
capture_output=True, text=True, timeout=600,
)
dump_time = time.monotonic() - t0
check(r.returncode == 0, "pg_dump failed:\n" + r.stdout + r.stderr)
size = out_file.stat().st_size
print(" dump size = %.1f MiB, time = %.2fs" % (size / 1024 / 1024, dump_time))
# новый контейнер как отдельное хранилище — в той же compose-сети,
# порт наружу не публикуем
restore_container = "mtuci-pg-restore"
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
subprocess.run(
["docker", "run", "-d", "--name", restore_container,
"--network", COMPOSE_NETWORK,
"-e", "POSTGRES_PASSWORD=restore", "-e", "POSTGRES_USER=postgres",
"-e", "POSTGRES_DB=high_load_messenger",
"postgres:18-alpine"],
capture_output=True, text=True,
)
# ждём готовности
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
c = subprocess.run(["docker", "exec", restore_container, "pg_isready", "-U", "postgres"],
capture_output=True, text=True)
if c.returncode == 0:
break
time.sleep(1)
else:
raise Failure("restore container не поднялся")
# pg_restore -c (clean) на пустую БД; restore-контейнер доступен по имени
# в compose-сети
t0 = time.monotonic()
r = subprocess.run(
["docker", "run", "--rm",
"--network", COMPOSE_NETWORK,
"-v", f"{EVIDENCE}:/in",
"postgres:18-alpine",
"pg_restore", "-d", "postgresql://postgres:restore@mtuci-pg-restore:5432/high_load_messenger",
"--no-owner", "--no-privileges", "/in/lab2_track2_dump.dump"],
capture_output=True, text=True, timeout=600,
)
restore_time = time.monotonic() - t0
print(" pg_restore exit=%d, time=%.2fs" % (r.returncode, restore_time))
check(r.returncode == 0, "pg_restore failed:\n" + r.stdout + r.stderr)
# Сверка: сравнение с исходной БД через docker exec psql внутри restore-контейнера
def q_restore(sql):
c = subprocess.run(["docker", "exec", restore_container, "psql", "-U", "postgres",
"-d", "high_load_messenger", "-tA", "-c", sql],
capture_output=True, text=True, timeout=120)
check(c.returncode == 0, "restore psql failed:\n" + c.stdout + c.stderr)
return c.stdout.strip().splitlines()[-1]
orig_total = int(_scalar(dsn, "SELECT count(*) FROM messages;"))
rest_total = int(q_restore("SELECT count(*) FROM messages;"))
check(orig_total == rest_total, "COUNT(*) mismatch: %d vs %d" % (orig_total, rest_total))
orig_dist = _scalar(dsn,
"SELECT string_agg(chat_id::text || ':' || c, ',' ORDER BY chat_id) "
"FROM (SELECT chat_id, count(*) c FROM messages GROUP BY chat_id) t;")
rest_dist = q_restore(
"SELECT string_agg(chat_id::text || ':' || c, ',' ORDER BY chat_id) "
"FROM (SELECT chat_id, count(*) c FROM messages GROUP BY chat_id) t;")
check(orig_dist == rest_dist, "распределение по чатам не совпадает")
# 100 случайных сообщений
ids_raw = _scalar(dsn, "SELECT string_agg(id::text, ',') FROM (SELECT id FROM messages ORDER BY random() LIMIT 100) t;")
ids = ids_raw.split(",")
check(len(ids) == 100, "не удалось выбрать 100 id")
ids_list = ",".join("'%s'" % i for i in ids)
orig_rows = _scalar(dsn,
"SELECT string_agg(id::text || '|' || text, E'\n' ORDER BY id) FROM messages WHERE id IN (%s);" % ids_list)
rest_rows = q_restore(
"SELECT string_agg(id::text || '|' || text, E'\n' ORDER BY id) FROM messages WHERE id IN (%s);" % ids_list)
check(orig_rows == rest_rows, "100 случайных сообщений различаются")
print(" OK: COUNT, распределение по чатам и 100 случайных сообщений совпадают")
print(" RTO (pg_restore) = %.2fs, dump = %.1f MiB" % (restore_time, size / 1024 / 1024))
(EVIDENCE / "lab2_track2_restore.txt").write_text(
"Track 2 restore\n"
"dump_time_s=%.3f\ndump_size_bytes=%d\nrestore_time_s=%.3f\n"
"count_original=%d\ncount_restored=%d\n"
"dist_match=true\nrandom100_match=true\n"
% (dump_time, size, restore_time, orig_total, rest_total),
encoding="utf-8")
# чистим restore-контейнер (volume анонимный)
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
return {"dump_time_s": round(dump_time, 3), "restore_time_s": round(restore_time, 3),
"dump_bytes": size, "count": orig_total}
# ---------- 5. RPO ----------
def step_rpo(dsn: str, base_url: str, ca_file):
print("=== 5. RPO: запись во время pg_dump ===")
c = Client(base_url, ca_file)
# Данные и сессии готовим заранее, до дампа.
alice, bob, eve, chat = c.bootstrap()
# Контроль: сессия должна быть валидна, иначе 401 — это не RPO.
me = c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
check(me["status"] == 200, "сессия должна быть валидна перед RPO-тестом")
marker = "rpo-" + uuid.uuid4().hex
# Запускаем pg_dump
out_file = EVIDENCE / "lab2_track2_rpo.dump"
if out_file.exists():
out_file.unlink()
# pg_dump: тот же приём, что в шаге 4, в compose-сети.
proc = subprocess.Popen(
["docker", "run", "--rm",
"--network", COMPOSE_NETWORK,
"-v", f"{EVIDENCE}:/out",
"postgres:18-alpine",
"pg_dump", "-Fc", "-f", "/out/lab2_track2_rpo.dump", dsn],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
)
time.sleep(1.0) # снапшот уже снят
# Свежий логин прямо перед записью: не зависим от TTL старой сессии.
cred = base64.b64encode(
(alice["registration"]["username"] + ":" +
alice["registration"]["password"]).encode()
).decode()
fresh = c.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": "Basic " + cred},
schema="Session")["json"]
alice_for_write = {"id": alice["id"], "user": alice["user"], "token": fresh["token"]}
written = c.message(alice_for_write, chat, marker)["json"]
code = proc.wait(timeout=300)
check(code == 0, "pg_dump для RPO упал")
# Временный restore-контейнер: в compose-сети, без публикации портов.
restore_container = "mtuci-pg-rpo"
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
subprocess.run(
["docker", "run", "-d", "--name", restore_container,
"--network", COMPOSE_NETWORK,
"-e", "POSTGRES_PASSWORD=restore", "-e", "POSTGRES_USER=postgres",
"-e", "POSTGRES_DB=high_load_messenger",
"postgres:18-alpine"],
capture_output=True, text=True,
)
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
c2 = subprocess.run(["docker", "exec", restore_container, "pg_isready", "-U", "postgres"],
capture_output=True, text=True)
if c2.returncode == 0:
break
time.sleep(1)
else:
raise Failure("restore container для RPO не поднялся")
# pg_restore идёт в restore-контейнер по имени в compose-сети.
r = subprocess.run(
["docker", "run", "--rm",
"--network", COMPOSE_NETWORK,
"-v", f"{EVIDENCE}:/in",
"postgres:18-alpine",
"pg_restore", "-d",
"postgresql://postgres:restore@mtuci-pg-rpo:5432/high_load_messenger",
"--no-owner", "--no-privileges", "/in/lab2_track2_rpo.dump"],
capture_output=True, text=True, timeout=600,
)
check(r.returncode == 0, "pg_restore для RPO упал:\n" + r.stdout + r.stderr)
c3 = subprocess.run(["docker", "exec", restore_container, "psql", "-U", "postgres",
"-d", "high_load_messenger", "-tA", "-c",
"SELECT count(*) FROM messages WHERE id = '%s';" % written["id"]],
capture_output=True, text=True)
present = int(c3.stdout.strip().splitlines()[-1])
check(present == 0, "сообщение, записанное во время дампа, попало в дамп (RPO не 0)")
print(" OK: сообщение %s записано после старта дампа и в restore отсутствует" % written["id"][:8])
(EVIDENCE / "lab2_track2_rpo.txt").write_text(
"Track 2 RPO\n"
"marker_message_id=%s\npresent_in_restore=%d\nexpected=0\n"
% (written["id"], present),
encoding="utf-8")
subprocess.run(["docker", "rm", "-f", restore_container], capture_output=True)
# ---------- main ----------
def main():
args = arguments("Track 2: plan, write cost, backup/restore, RPO")
dsn = _read_dsn()
total = step_seed(dsn)
step_explain(dsn, total)
step_write_cost(dsn)
step_backup_restore(dsn)
step_rpo(dsn, args.base_url, args.ca_file)
print("PASS: track 2 (explain + write cost + backup/restore + RPO)")
if __name__ == "__main__":
run(main)