forked from mtusi2026/mtusi2026lab2
2
This commit is contained in:
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
# Логический backup рабочей БД стенда (Compose): pg_dump -Fc из контейнера postgres в backups/ (каталог в .gitignore).
|
||||
# Снапшот согласован на момент НАЧАЛА pg_dump; запись при этом не блокируется. Секреты в вывод не попадают.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p backups && umask 077
|
||||
OUT="backups/messenger-$(date -u +%Y%m%dT%H%M%SZ).dump"
|
||||
docker compose --env-file .env exec -T postgres pg_dump -U messenger -d messenger -Fc --no-owner > "$OUT"
|
||||
echo "$OUT $(wc -c < "$OUT") bytes"
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
# Доказательство, что restart-hook реально перезапустил процессы: StartedAt контейнеров ДО и ПОСЛЕ + прогон теста.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
DC="docker compose --env-file .env"
|
||||
show() { for s in postgres valkey api; do printf '%-9s StartedAt=%s\n' "$s" "$(docker inspect -f '{{.State.StartedAt}}' "$($DC ps -q $s)")"; done; }
|
||||
echo "== before =="; show; $DC ps
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
echo "== after =="; show; $DC ps
|
||||
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Обоснование индекса истории: EXPLAIN (ANALYZE, BUFFERS) на синтетических данных.
|
||||
|
||||
Использует ОТДЕЛЬНУЮ БД (миграции применяются к ней же): python scripts/explain_history.py
|
||||
Переменные: DATABASE_URL_FILE/DATABASE_URL указывают на БД для эксперимента (не на рабочую!).
|
||||
Данные: 1 «целевой» чат (TARGET_N сообщений) + 4 чата-шума по NOISE_N — таблица ≈ 5×, как в реальной системе.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
import psycopg # noqa: E402
|
||||
|
||||
from app import config, migrate # noqa: E402
|
||||
|
||||
TARGET_N = int(os.environ.get("TARGET_N", 20000))
|
||||
NOISE_N = int(os.environ.get("NOISE_N", 20000))
|
||||
PAGE = 50
|
||||
|
||||
|
||||
def main():
|
||||
migrate.apply_all(config.DATABASE_URL)
|
||||
with psycopg.connect(config.DATABASE_URL, autocommit=True) as conn:
|
||||
conn.execute("TRUNCATE messages, chat_members, chats, sessions, users CASCADE")
|
||||
user = uuid.uuid4()
|
||||
conn.execute("INSERT INTO users (id, username, display_name, password_hash) VALUES (%s,'explain_user','x','x')", (user,))
|
||||
chats = [uuid.uuid4() for _ in range(5)]
|
||||
for cid in chats:
|
||||
conn.execute("INSERT INTO chats (id, title, created_by) VALUES (%s,'t',%s)", (cid, user))
|
||||
conn.execute("INSERT INTO chat_members (chat_id, user_id) VALUES (%s,%s)", (cid, user))
|
||||
# Вставка вперемешку по времени (как у живых чатов): created_at — равномерная шкала + микросекунды.
|
||||
for i, cid in enumerate(chats):
|
||||
n = TARGET_N if i == 0 else NOISE_N
|
||||
conn.execute("""INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), %s, %s, 'message ' || g,
|
||||
timestamptz '2026-01-01 00:00:00+00' + (g * 5 + %s) * interval '1 second' + (random()*1000) * interval '1 microsecond'
|
||||
FROM generate_series(1, %s) g""", (cid, user, i, n))
|
||||
conn.execute("VACUUM (ANALYZE) messages")
|
||||
target = chats[0]
|
||||
total = conn.execute("SELECT count(*) FROM messages").fetchone()[0]
|
||||
in_chat = conn.execute("SELECT count(*) FROM messages WHERE chat_id=%s", (target,)).fetchone()[0]
|
||||
# Курсор «глубоко в истории»: элемент №15000 по (created_at, id).
|
||||
ts, mid = conn.execute("""SELECT created_at, id FROM messages WHERE chat_id=%s
|
||||
ORDER BY created_at, id OFFSET %s LIMIT 1""", (target, min(15000, in_chat - PAGE - 1))).fetchone()
|
||||
|
||||
def explain(title, sql, params, pre=None, post=None):
|
||||
print("\n" + "=" * 100 + "\n" + title + "\n" + "=" * 100)
|
||||
with conn.transaction(force_rollback=True): # DDL в PostgreSQL транзакционен: индекс вернётся
|
||||
if pre:
|
||||
conn.execute(pre)
|
||||
for line in conn.execute("EXPLAIN (ANALYZE, BUFFERS) " + sql, params).fetchall():
|
||||
print(line[0])
|
||||
|
||||
print("rows total=%d, rows in target chat=%d, page=%d" % (total, in_chat, PAGE))
|
||||
print(conn.execute("SELECT version()").fetchone()[0])
|
||||
first = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id=%s ORDER BY created_at, id LIMIT %s"
|
||||
keyset = ("SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id=%s "
|
||||
"AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s")
|
||||
offset = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id=%s ORDER BY created_at, id OFFSET 15000 LIMIT %s"
|
||||
drop = "DROP INDEX messages_chat_created_id_idx"
|
||||
explain("A. Первая страница, keyset, С индексом (chat_id, created_at, id)", first, (target, PAGE + 1))
|
||||
explain("B. Страница на глубине ~15 000, keyset, С индексом", keyset, (target, ts, mid, PAGE + 1))
|
||||
explain("C. Та же страница, keyset, БЕЗ индекса (индекс удалён в откатываемой транзакции)", keyset, (target, ts, mid, PAGE + 1), pre=drop)
|
||||
explain("D. Та же глубина через OFFSET 15000, С индексом (антипример)", offset, (target, PAGE + 1))
|
||||
explain("E. Первая страница, БЕЗ индекса", first, (target, PAGE + 1), pre=drop)
|
||||
print("\nИндексы messages:")
|
||||
for row in conn.execute("SELECT indexname, indexdef FROM pg_indexes WHERE tablename='messages' ORDER BY 1").fetchall():
|
||||
print(" ", row[0], "->", row[1])
|
||||
print("Размер индекса: ", conn.execute("SELECT pg_size_pretty(pg_relation_size('messages_chat_created_id_idx'))").fetchone()[0],
|
||||
"; таблицы:", conn.execute("SELECT pg_size_pretty(pg_relation_size('messages'))").fetchone()[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
# Создаёт локальные секреты (случайные пароли) и .env. Идемпотентно: существующие секреты НЕ перезаписываются
|
||||
# (иначе пароль в уже инициализированном томе PostgreSQL перестал бы совпадать). Каталог secrets/ в .gitignore.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
mkdir -p secrets
|
||||
rand() { if command -v openssl >/dev/null 2>&1; then openssl rand -hex 24; else python3 -c 'import secrets;print(secrets.token_hex(24))'; fi; }
|
||||
put() { [ -f "secrets/$1" ] || { printf '%s' "$2" > "secrets/$1"; echo "created secrets/$1"; }; }
|
||||
[ -f secrets/postgres_password ] || printf '%s' "$(rand)" > secrets/postgres_password
|
||||
[ -f secrets/valkey_password ] || printf '%s' "$(rand)" > secrets/valkey_password
|
||||
PG=$(cat secrets/postgres_password); VK=$(cat secrets/valkey_password)
|
||||
put database_url "postgresql://messenger:${PG}@postgres:5432/messenger"
|
||||
put cache_url "redis://:${VK}@valkey:6379/0"
|
||||
# Файлы читаются процессами контейнеров с другим UID, поэтому 644; доступ ограничивает каталог secrets/ (700).
|
||||
chmod 700 secrets && chmod 644 secrets/*
|
||||
[ -f .env ] || { cp .env.example .env; echo "created .env"; }
|
||||
Executable → Regular
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
# Hook для tests/resilience.py --mode persistence: перезапускает API и ВСЕ хранилища (PostgreSQL, Valkey),
|
||||
# тома сохраняются. Никаких down -v / flush / повторного seed. Вывод скрыт тестом, секреты не печатаются.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
DC="docker compose --env-file .env"
|
||||
$DC stop api valkey postgres # SIGTERM; compose сам останавливает в обратном порядке зависимостей
|
||||
$DC up -d # поднимает postgres+valkey, повторно гонит idempotent-миграции, затем api
|
||||
i=0
|
||||
until [ "$(docker inspect -f '{{.State.Health.Status}}' "$($DC ps -q api)" 2>/dev/null || true)" = "healthy" ]; do
|
||||
i=$((i + 1))
|
||||
[ "$i" -le 70 ] || { echo "api did not become healthy" >&2; exit 1; }
|
||||
sleep 1
|
||||
done
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/bin/sh
|
||||
# Восстановление dump в НОВОЕ хранилище (временный контейнер со свежим томом) и сверка с рабочей БД.
|
||||
# Использование: sh scripts/restore-check.sh backups/messenger-XXXX.dump
|
||||
# ВНИМАНИЕ: не запускался в среде подготовки (там нет Docker); эквивалентный эксперимент выполнен через
|
||||
# scripts/track2_restore.py на двух отдельных серверах PostgreSQL (см. evidence/track2-restore.json).
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
DUMP="${1:?usage: restore-check.sh <dump>}"
|
||||
DC="docker compose --env-file .env"
|
||||
NAME="pg-restore-check-$$"
|
||||
trap 'docker rm -f "$NAME" >/dev/null 2>&1 || true' EXIT
|
||||
docker run -d --name "$NAME" -e POSTGRES_PASSWORD=restore-check-only -e POSTGRES_USER=messenger -e POSTGRES_DB=restored postgres:17.6-alpine >/dev/null
|
||||
i=0; until docker exec "$NAME" pg_isready -U messenger -d restored >/dev/null 2>&1; do i=$((i+1)); [ $i -le 60 ] || exit 1; sleep 1; done
|
||||
START=$(date +%s)
|
||||
docker exec -i "$NAME" pg_restore -U messenger -d restored --no-owner --exit-on-error < "$DUMP"
|
||||
docker exec "$NAME" psql -U messenger -d restored -qc "ANALYZE"
|
||||
echo "restore+analyze seconds: $(( $(date +%s) - START ))"
|
||||
Q="select 'users',count(*) from users union all select 'chats',count(*) from chats union all select 'chat_members',count(*) from chat_members union all select 'messages',count(*) from messages union all select 'messages_md5', md5(string_agg(id::text||chat_id::text||sender_id::text||created_at::text||md5(text), ',' order by id))::text from messages"
|
||||
echo "--- restored ---"; docker exec "$NAME" psql -U messenger -d restored -At -c "$Q"
|
||||
echo "--- source (rows after the dump may differ: that is the RPO window) ---"
|
||||
$DC exec -T postgres psql -U messenger -d messenger -At -c "$Q"
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
# Проверка expiry: пересоздаёт api с SESSION_TTL_SECONDS=2, гоняет tests_student/test_expiry.py, возвращает штатный TTL.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
DC="docker compose --env-file .env"
|
||||
wait_healthy() { i=0; until [ "$(docker inspect -f '{{.State.Health.Status}}' "$($DC ps -q api)")" = "healthy" ]; do i=$((i+1)); [ $i -le 60 ] || exit 1; sleep 1; done; }
|
||||
trap 'SESSION_TTL_SECONDS= $DC up -d --force-recreate api >/dev/null; wait_healthy' EXIT
|
||||
SESSION_TTL_SECONDS=2 $DC up -d --force-recreate api; wait_healthy
|
||||
(cd tests_student && python3 test_expiry.py 2)
|
||||
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Трек 2 (часть A): план запроса истории на ≥100 000 сообщений с НЕРАВНОМЕРНЫМИ чатами.
|
||||
|
||||
Гипотеза H1: составной индекс (chat_id, created_at, id) даёт страницу истории за O(размер страницы)
|
||||
для любого чата и любой глубины, а индекс только по chat_id или «глобальный» (created_at, id) — нет;
|
||||
покрывающий вариант (INCLUDE) читает меньше страниц, но платит за это размером индекса и стоимостью записи.
|
||||
Гипотеза H2: каждый дополнительный индекс заметно удорожает вставку, и цена растёт с шириной индекса.
|
||||
|
||||
Запуск (ОТДЕЛЬНАЯ БД эксперимента, не рабочая!):
|
||||
DATABASE_URL_FILE=/path/to/url python scripts/track2_indexes.py --out evidence/track2-indexes.json
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
import psycopg # noqa: E402
|
||||
|
||||
from app import config, migrate # noqa: E402
|
||||
|
||||
# Неравномерные размеры: 1 гигант, 2 больших, 10 средних, 50 малых, 500 «почти пустых» = ровно 100 000.
|
||||
SIZES = [50000, 20000, 10000] + [1000] * 10 + [100] * 50 + [10] * 500
|
||||
CONFIGS = [
|
||||
("C0_pk_only", None),
|
||||
("C1_chat_id", "CREATE INDEX hist_idx ON messages (chat_id)"),
|
||||
("C2_chat_created_id", "CREATE INDEX hist_idx ON messages (chat_id, created_at, id)"),
|
||||
("C3_covering_include", "CREATE INDEX hist_idx ON messages (chat_id, created_at, id) INCLUDE (sender_id, text)"),
|
||||
("C4_global_created_id", "CREATE INDEX hist_idx ON messages (created_at, id)"),
|
||||
]
|
||||
COLS = "id, chat_id, sender_id, text, created_at"
|
||||
PAGE = 50
|
||||
READ_RUNS, WARMUP = 300, 20
|
||||
SINGLE_INSERTS = 2000
|
||||
BULK_REPEATS = 3
|
||||
|
||||
|
||||
def pct(values, p):
|
||||
values = sorted(values)
|
||||
return values[max(0, min(len(values) - 1, int(round(p * len(values) + 0.5)) - 1))]
|
||||
|
||||
|
||||
def seed_source(conn):
|
||||
"""Один раз генерируем данные в messages_src: тексты разной длины (10–400 символов), время по возрастанию в чате."""
|
||||
conn.execute("TRUNCATE messages, chat_members, chats, sessions, users CASCADE")
|
||||
conn.execute("DROP TABLE IF EXISTS messages_src")
|
||||
user = uuid.uuid4()
|
||||
conn.execute("INSERT INTO users (id, username, display_name, password_hash) VALUES (%s,'track2_user','x','x')", (user,))
|
||||
chats = [uuid.uuid4() for _ in SIZES]
|
||||
for cid in chats:
|
||||
conn.execute("INSERT INTO chats (id, title, created_by) VALUES (%s,'t',%s)", (cid, user))
|
||||
conn.execute("INSERT INTO chat_members (chat_id, user_id) VALUES (%s,%s)", (cid, user))
|
||||
conn.execute("""CREATE TABLE messages_src AS
|
||||
SELECT gen_random_uuid() AS id, c.chat_id, %s::uuid AS sender_id,
|
||||
left(repeat('сообщение ', 60), 10 + (random() * 390)::int) AS text,
|
||||
timestamptz '2026-01-01 00:00:00+00' + (g.n * 7 + c.idx) * interval '1 second'
|
||||
+ (random() * 1000) * interval '1 microsecond' AS created_at
|
||||
FROM (SELECT unnest(%s::uuid[]) AS chat_id, unnest(%s::int[]) AS size, generate_series(1, %s) AS idx) c
|
||||
CROSS JOIN LATERAL generate_series(1, c.size) AS g(n)""", (user, chats, SIZES, len(SIZES)))
|
||||
conn.execute("ANALYZE messages_src")
|
||||
return user, chats
|
||||
|
||||
|
||||
def reset_history_index(conn, ddl):
|
||||
conn.execute("DROP INDEX IF EXISTS messages_chat_created_id_idx")
|
||||
conn.execute("DROP INDEX IF EXISTS hist_idx")
|
||||
if ddl:
|
||||
conn.execute(ddl)
|
||||
|
||||
|
||||
def measure_writes(conn, ddl, chats):
|
||||
"""Стоимость записи: (1) пакетная загрузка 100 000 строк, (2) 2000 одиночных INSERT с коммитом (как работает API)."""
|
||||
bulk = []
|
||||
for _ in range(BULK_REPEATS):
|
||||
conn.execute("TRUNCATE messages")
|
||||
reset_history_index(conn, ddl)
|
||||
t0 = time.perf_counter()
|
||||
conn.execute("INSERT INTO messages (id, chat_id, sender_id, text, created_at) SELECT * FROM messages_src")
|
||||
bulk.append(time.perf_counter() - t0)
|
||||
conn.execute("VACUUM (ANALYZE) messages")
|
||||
# Одиночные вставки: чаты выбираются пропорционально размеру (больше сообщений — чаще пишут), autocommit = COMMIT на каждую.
|
||||
import random
|
||||
rnd = random.Random(42)
|
||||
sender = conn.execute("SELECT sender_id FROM messages LIMIT 1").fetchone()[0]
|
||||
singles = []
|
||||
for _ in range(SINGLE_INSERTS):
|
||||
cid = rnd.choices(chats, weights=SIZES)[0]
|
||||
t0 = time.perf_counter()
|
||||
conn.execute("INSERT INTO messages (id, chat_id, sender_id, text) VALUES (%s,%s,%s,%s)",
|
||||
(uuid.uuid4(), cid, sender, "одиночная вставка " + "x" * rnd.randint(10, 300)))
|
||||
singles.append(time.perf_counter() - t0)
|
||||
conn.execute("VACUUM (ANALYZE) messages")
|
||||
size = lambda rel: conn.execute("SELECT pg_relation_size(%s)", (rel,)).fetchone()[0]
|
||||
return {
|
||||
"bulk_insert_100k_seconds": {"runs": [round(x, 3) for x in bulk], "median": round(statistics.median(bulk), 3)},
|
||||
"single_insert_ms": {"n": len(singles), "mean": round(1000 * statistics.mean(singles), 3),
|
||||
"p50": round(1000 * pct(singles, .50), 3), "p95": round(1000 * pct(singles, .95), 3),
|
||||
"p99": round(1000 * pct(singles, .99), 3)},
|
||||
"history_index_bytes": size("hist_idx") if ddl else 0,
|
||||
"table_bytes": size("messages"),
|
||||
"pk_index_bytes": size("messages_pkey"),
|
||||
}
|
||||
|
||||
|
||||
def plan_summary(conn, sql, params):
|
||||
plan = conn.execute("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + sql, params).fetchone()[0][0]
|
||||
nodes, stack = [], [plan["Plan"]]
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
nodes.append(node["Node Type"])
|
||||
stack.extend(node.get("Plans", []))
|
||||
root = plan["Plan"]
|
||||
return {"nodes": nodes[::-1], "execution_ms": round(plan["Execution Time"], 3),
|
||||
"shared_buffers": root.get("Shared Hit Blocks", 0) + root.get("Shared Read Blocks", 0),
|
||||
"rows": root["Actual Rows"]}
|
||||
|
||||
|
||||
def measure_reads(conn, queries):
|
||||
out = {}
|
||||
for name, (sql, params) in queries.items():
|
||||
for _ in range(WARMUP):
|
||||
conn.execute(sql, params).fetchall()
|
||||
times = []
|
||||
for _ in range(READ_RUNS):
|
||||
t0 = time.perf_counter()
|
||||
conn.execute(sql, params).fetchall()
|
||||
times.append(time.perf_counter() - t0)
|
||||
out[name] = {"p50_ms": round(1000 * pct(times, .50), 3), "p95_ms": round(1000 * pct(times, .95), 3),
|
||||
"plan": plan_summary(conn, sql, params)}
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default="evidence/track2-indexes.json")
|
||||
ap.add_argument("--reverse", action="store_true", help="обратный порядок конфигураций: проверка, что результат не зависит от порядка/прогрева")
|
||||
args = ap.parse_args()
|
||||
migrate.apply_all(config.DATABASE_URL)
|
||||
with psycopg.connect(config.DATABASE_URL, autocommit=True) as conn:
|
||||
user, chats = seed_source(conn)
|
||||
total = conn.execute("SELECT count(*) FROM messages_src").fetchone()[0]
|
||||
big, small, mid = chats[0], chats[-1], chats[3] # 50 000 / 10 / 1 000 сообщений
|
||||
deep_ts, deep_id = conn.execute("SELECT created_at, id FROM messages_src WHERE chat_id=%s ORDER BY created_at, id OFFSET 40000 LIMIT 1", (big,)).fetchone()
|
||||
mid_ts, mid_id = conn.execute("SELECT created_at, id FROM messages_src WHERE chat_id=%s ORDER BY created_at, id OFFSET 500 LIMIT 1", (mid,)).fetchone()
|
||||
first = "SELECT %s FROM messages WHERE chat_id=%%s ORDER BY created_at, id LIMIT %%s" % COLS
|
||||
keyset = "SELECT %s FROM messages WHERE chat_id=%%s AND (created_at, id) > (%%s, %%s) ORDER BY created_at, id LIMIT %%s" % COLS
|
||||
offset = "SELECT %s FROM messages WHERE chat_id=%%s ORDER BY created_at, id OFFSET 40000 LIMIT %%s" % COLS
|
||||
queries = {
|
||||
"Q1_first_page_giant_50k": (first, (big, PAGE + 1)),
|
||||
"Q2_deep_keyset_giant_at_40k": (keyset, (big, deep_ts, deep_id, PAGE + 1)),
|
||||
"Q3_first_page_tiny_10": (first, (small, PAGE + 1)),
|
||||
"Q4_keyset_mid_1k_at_500": (keyset, (mid, mid_ts, mid_id, PAGE + 1)),
|
||||
"Q5_OFFSET_40000_giant": (offset, (big, PAGE + 1)),
|
||||
}
|
||||
env = {
|
||||
"postgres": conn.execute("SHOW server_version").fetchone()[0],
|
||||
"shared_buffers": conn.execute("SHOW shared_buffers").fetchone()[0],
|
||||
"work_mem": conn.execute("SHOW work_mem").fetchone()[0],
|
||||
"fsync": conn.execute("SHOW fsync").fetchone()[0],
|
||||
"synchronous_commit": conn.execute("SHOW synchronous_commit").fetchone()[0],
|
||||
"cpus": os.cpu_count(), "python": platform.python_version(), "platform": platform.platform(),
|
||||
}
|
||||
results = {"environment": env, "dataset": {"messages": total, "chats": len(SIZES), "chat_size_distribution":
|
||||
{"50000": 1, "20000": 1, "10000": 1, "1000": 10, "100": 50, "10": 500}, "text_length_chars": "10..400 (равномерно)",
|
||||
"page_size": PAGE, "read_runs": READ_RUNS, "warmup": WARMUP, "single_inserts": SINGLE_INSERTS,
|
||||
"bulk_repeats": BULK_REPEATS, "order": "reverse" if args.reverse else "forward"}, "configs": {}}
|
||||
for name, ddl in (reversed(CONFIGS) if args.reverse else CONFIGS):
|
||||
print("== %s ==" % name, flush=True)
|
||||
w = measure_writes(conn, ddl, chats)
|
||||
r = measure_reads(conn, queries)
|
||||
results["configs"][name] = {"ddl": ddl, "writes": w, "reads": r}
|
||||
print(" bulk %.2fs, single p50 %.3f ms, index %.1f MiB; Q1 p50 %.3f ms, Q2 p50 %.3f ms, Q3 p50 %.3f ms" % (
|
||||
w["bulk_insert_100k_seconds"]["median"], w["single_insert_ms"]["p50"], w["history_index_bytes"] / 2**20,
|
||||
r["Q1_first_page_giant_50k"]["p50_ms"], r["Q2_deep_keyset_giant_at_40k"]["p50_ms"], r["Q3_first_page_tiny_10"]["p50_ms"]), flush=True)
|
||||
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.out).write_text(json.dumps(results, ensure_ascii=False, indent=2) + "\n")
|
||||
print("saved", args.out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Трек 2 (часть B): backup -> восстановление в НОВОЕ хранилище -> сверка количества и выбранных сообщений.
|
||||
|
||||
Источник — БД эксперимента (после track2_indexes.py, 100 000+ сообщений). Приёмник — другой кластер/сервер PostgreSQL
|
||||
(новый каталог данных, другой порт), т.е. настоящее «новое хранилище», а не вторая БД в том же кластере.
|
||||
|
||||
python scripts/track2_restore.py --src-url URL --dst-admin-url URL(postgres db на новом сервере) \
|
||||
--dst-db messenger_restored --pg-bin /usr/lib/postgresql/16/bin --dump-file /tmp/t2.dump --out evidence/track2-restore.json
|
||||
|
||||
Что измеряется: длительность pg_dump и pg_restore (+ ANALYZE), размер дампа, полнота сверки, окно потери данных (RPO):
|
||||
записи, сделанные ПОСЛЕ начала дампа, в копии отсутствуют; отозванная после дампа сессия в копии «воскресает».
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
import psycopg
|
||||
|
||||
|
||||
def with_db(url, db):
|
||||
parts = urlsplit(url)
|
||||
return urlunsplit(parts._replace(path="/" + db))
|
||||
|
||||
|
||||
def facts(conn):
|
||||
"""Отпечаток состояния: счётчики, md5 содержимого, схема, выбранные строки."""
|
||||
one = lambda sql, *p: conn.execute(sql, p).fetchone()[0]
|
||||
out = {"counts": {t: one("SELECT count(*) FROM " + t) for t in ("users", "chats", "chat_members", "messages", "sessions")}}
|
||||
out["messages_md5"] = one("""SELECT md5(string_agg(id::text || '|' || chat_id::text || '|' || sender_id::text || '|' ||
|
||||
created_at::text || '|' || md5(text), ',' ORDER BY id)) FROM messages""")
|
||||
out["per_chat_counts_md5"] = one("SELECT md5(string_agg(chat_id::text || ':' || n, ',' ORDER BY chat_id)) FROM (SELECT chat_id, count(*) n FROM messages GROUP BY chat_id) s")
|
||||
out["indexes_md5"] = one("SELECT md5(string_agg(indexname || indexdef, E'\\n' ORDER BY indexname)) FROM pg_indexes WHERE schemaname='public'")
|
||||
out["constraints_md5"] = one("SELECT md5(string_agg(conname || pg_get_constraintdef(oid), E'\\n' ORDER BY conname)) FROM pg_constraint WHERE connamespace = 'public'::regnamespace")
|
||||
out["migrations"] = [r[0] for r in conn.execute("SELECT version FROM schema_migrations ORDER BY 1").fetchall()]
|
||||
big = one("SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) DESC, chat_id LIMIT 1")
|
||||
picks = {
|
||||
"first_overall": "SELECT * FROM messages ORDER BY created_at, id LIMIT 1",
|
||||
"last_overall": "SELECT * FROM messages ORDER BY created_at DESC, id DESC LIMIT 1",
|
||||
"giant_chat_item_15000": "SELECT * FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET 15000 LIMIT 1" % big,
|
||||
"longest_text": "SELECT * FROM messages ORDER BY char_length(text) DESC, id LIMIT 1",
|
||||
"shortest_text": "SELECT * FROM messages ORDER BY char_length(text), id LIMIT 1",
|
||||
"pseudo_random_by_md5": "SELECT * FROM messages ORDER BY md5(id::text) LIMIT 1",
|
||||
}
|
||||
out["selected_messages"] = {k: one("SELECT row_to_json(s)::text FROM (%s) s" % q) for k, q in picks.items()}
|
||||
return out
|
||||
|
||||
|
||||
def run(cmd, env):
|
||||
t0 = time.perf_counter()
|
||||
proc = subprocess.run(cmd, env=env, capture_output=True, text=True)
|
||||
dt = time.perf_counter() - t0
|
||||
if proc.returncode:
|
||||
raise SystemExit("command failed (%s): %s" % (cmd[0], proc.stderr[-800:]))
|
||||
return dt
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--src-url", required=True)
|
||||
ap.add_argument("--dst-admin-url", required=True, help="URL БД postgres на НОВОМ сервере (создание БД восстановления)")
|
||||
ap.add_argument("--dst-db", default="messenger_restored")
|
||||
ap.add_argument("--pg-bin", default="/usr/lib/postgresql/16/bin")
|
||||
ap.add_argument("--dump-file", default="/tmp/track2.dump")
|
||||
ap.add_argument("--out", default="evidence/track2-restore.json")
|
||||
a = ap.parse_args()
|
||||
dst_url = with_db(a.dst_admin_url, a.dst_db)
|
||||
env = dict(os.environ)
|
||||
|
||||
with psycopg.connect(a.src_url, autocommit=True) as src:
|
||||
# 1. Приводим источник к «боевой» схеме: без вспомогательной таблицы, канонический индекс истории, одна сессия.
|
||||
src.execute("DROP TABLE IF EXISTS messages_src")
|
||||
src.execute("DROP INDEX IF EXISTS hist_idx")
|
||||
src.execute("CREATE INDEX IF NOT EXISTS messages_chat_created_id_idx ON messages (chat_id, created_at, id)")
|
||||
user = src.execute("SELECT id FROM users LIMIT 1").fetchone()[0]
|
||||
token_hash = os.urandom(32)
|
||||
src.execute("INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s,%s, now() + interval '1 day')", (token_hash, user))
|
||||
src.execute("VACUUM (ANALYZE)")
|
||||
before = facts(src)
|
||||
db_bytes = src.execute("SELECT pg_database_size(current_database())").fetchone()[0]
|
||||
|
||||
# 2. Backup: логический, custom-формат; снапшот согласован на момент НАЧАЛА pg_dump.
|
||||
t_dump = run([a.pg_bin + "/pg_dump", "-Fc", "--no-owner", "-f", a.dump_file, "-d", a.src_url], env)
|
||||
dump_bytes = Path(a.dump_file).stat().st_size
|
||||
|
||||
# 3. Окно потери (RPO): изменения ПОСЛЕ дампа — 500 новых сообщений и отзыв сессии.
|
||||
chat = src.execute("SELECT chat_id FROM messages LIMIT 1").fetchone()[0]
|
||||
sender = src.execute("SELECT sender_id FROM messages WHERE chat_id=%s LIMIT 1", (chat,)).fetchone()[0]
|
||||
for i in range(500):
|
||||
src.execute("INSERT INTO messages (id, chat_id, sender_id, text) VALUES (%s,%s,%s,%s)", (uuid.uuid4(), chat, sender, "после бэкапа %d" % i))
|
||||
src.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,))
|
||||
after_source = facts(src)
|
||||
|
||||
# 4. Новое хранилище: другая БД на другом сервере.
|
||||
with psycopg.connect(a.dst_admin_url, autocommit=True) as admin:
|
||||
admin.execute('DROP DATABASE IF EXISTS "%s"' % a.dst_db)
|
||||
admin.execute('CREATE DATABASE "%s"' % a.dst_db)
|
||||
t_restore = run([a.pg_bin + "/pg_restore", "--no-owner", "--exit-on-error", "-d", dst_url, a.dump_file], env)
|
||||
with psycopg.connect(dst_url, autocommit=True) as dst:
|
||||
t0 = time.perf_counter()
|
||||
dst.execute("ANALYZE") # pg_dump не переносит статистику планировщика: без ANALYZE планы могут быть плохими
|
||||
t_analyze = time.perf_counter() - t0
|
||||
restored = facts(dst)
|
||||
plans = {}
|
||||
big = dst.execute("SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) DESC, chat_id LIMIT 1").fetchone()[0]
|
||||
plan = dst.execute("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) SELECT id, chat_id, sender_id, text, created_at FROM messages "
|
||||
"WHERE chat_id=%s ORDER BY created_at, id LIMIT 51", (big,)).fetchone()[0][0]
|
||||
node = plan["Plan"]
|
||||
while node.get("Plans"):
|
||||
node = node["Plans"][0]
|
||||
plans["first_page_giant"] = {"node": node["Node Type"], "index": node.get("Index Name"), "execution_ms": round(plan["Execution Time"], 3)}
|
||||
session_present = dst.execute("SELECT count(*) FROM sessions WHERE token_hash=%s", (token_hash,)).fetchone()[0]
|
||||
|
||||
# 5. Сверка: копия обязана совпасть с источником НА МОМЕНТ ДАМПА.
|
||||
checks = {k: before[k] == restored[k] for k in before}
|
||||
checks["selected_messages"] = {k: before["selected_messages"][k] == restored["selected_messages"][k] for k in before["selected_messages"]}
|
||||
ok = all(v if isinstance(v, bool) else all(v.values()) for v in checks.values())
|
||||
rpo = {
|
||||
"messages_lost_in_restore": after_source["counts"]["messages"] - restored["counts"]["messages"],
|
||||
"revoked_session_resurrected_in_restore": bool(session_present),
|
||||
}
|
||||
result = {
|
||||
"snapshot_consistent_with_restore": ok, "checks": checks, "rpo_window_observed": rpo,
|
||||
"timing_seconds": {"pg_dump": round(t_dump, 2), "pg_restore": round(t_restore, 2), "analyze_after_restore": round(t_analyze, 2),
|
||||
"rto_dump_free_restore_plus_analyze": round(t_restore + t_analyze, 2)},
|
||||
"sizes_bytes": {"source_database": db_bytes, "dump_file": dump_bytes},
|
||||
"counts_at_dump": before["counts"], "counts_restored": restored["counts"], "counts_source_after_extra_writes": after_source["counts"],
|
||||
"restored_plan": plans,
|
||||
}
|
||||
Path(a.out).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(a.out).write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
raise SystemExit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user