Лабораторная 2: PostgreSQL, сессии, Valkey, трек 2

This commit is contained in:
2026-10-16 19:39:21 +03:00
commit aa0fd15616
82 changed files with 7996 additions and 0 deletions
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
# Логический backup рабочей БД стенда (Compose): pg_dump -Fc из контейнера postgres в backups/ (каталог в .gitignore).
# Снапшот согласован на момент НАЧАЛА pg_dump; запись при этом не блокируется. Секреты в вывод не попадают.
set -eu
cd "$(dirname "$0")/.."
mkdir -p backups && umask 077
OUT="backups/messenger-$(date -u +%Y%m%dT%H%M%SZ).dump"
docker compose --env-file .env exec -T postgres pg_dump -U messenger -d messenger -Fc --no-owner > "$OUT"
echo "$OUT $(wc -c < "$OUT") bytes"
+54
View File
@@ -0,0 +1,54 @@
#!/usr/bin/env python3
"""Offline structural checks; full schema validation is a separate target."""
import ast
import json
from pathlib import Path
import re
ROOT = Path(__file__).resolve().parents[1]
def main():
lab = json.loads((ROOT/'lab.json').read_text())['number']
spec = json.loads((ROOT/'contracts/openapi.json').read_text())
assert spec['openapi']=='3.1.0'
assert spec['info']['version']=='1.%d.0'%lab
operation_ids = set()
def walk(value):
if isinstance(value,dict):
if '$ref' in value:
pointer = value['$ref']
assert pointer.startswith('#/'), 'unexpected external reference'
target = spec
for part in pointer[2:].split('/'):
target = target[part.replace('~1','/').replace('~0','~')]
for v in value.values():
walk(v)
elif isinstance(value,list):
for v in value:
walk(v)
walk(spec)
for route,methods in spec['paths'].items():
placeholders = set(re.findall(r'\{([^}]+)\}',route))
for method,op in methods.items():
assert op['operationId'] not in operation_ids, 'duplicate operationId'
operation_ids.add(op['operationId'])
params = {p['name'] for p in op.get('parameters',[]) if p['in']=='path' and p.get('required')}
assert placeholders==params, 'path parameter mismatch: '+route
for response in op['responses'].values():
assert 'X-Request-Id' in response['headers']
if lab>=3:
assert 'X-Instance-Id' in response['headers']
assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
ast.parse(file.read_text(),filename=str(file))
for file in ROOT.rglob('*.json'):
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
continue
json.loads(file.read_text())
for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
assert (ROOT/required).is_file(), 'missing '+required
print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
if __name__=='__main__':
main()
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
# Доказательство, что restart-hook реально перезапустил процессы: StartedAt контейнеров ДО и ПОСЛЕ + прогон теста.
set -eu
cd "$(dirname "$0")/.."
DC="docker compose --env-file .env"
show() { for s in postgres valkey api; do printf '%-9s StartedAt=%s\n' "$s" "$(docker inspect -f '{{.State.StartedAt}}' "$($DC ps -q $s)")"; done; }
echo "== before =="; show; $DC ps
make test-persistence ACTION_SCRIPT=scripts/restart.sh
echo "== after =="; show; $DC ps
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env python3
"""Обоснование индекса истории: EXPLAIN (ANALYZE, BUFFERS) на синтетических данных.
Использует ОТДЕЛЬНУЮ БД (миграции применяются к ней же): python scripts/explain_history.py
Переменные: DATABASE_URL_FILE/DATABASE_URL указывают на БД для эксперимента (не на рабочую!).
Данные: 1 «целевой» чат (TARGET_N сообщений) + 4 чата-шума по NOISE_N — таблица ≈ 5×, как в реальной системе.
"""
import os
import sys
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import psycopg # noqa: E402
from app import config, migrate # noqa: E402
TARGET_N = int(os.environ.get("TARGET_N", 20000))
NOISE_N = int(os.environ.get("NOISE_N", 20000))
PAGE = 50
def main():
migrate.apply_all(config.DATABASE_URL)
with psycopg.connect(config.DATABASE_URL, autocommit=True) as conn:
conn.execute("TRUNCATE messages, chat_members, chats, sessions, users CASCADE")
user = uuid.uuid4()
conn.execute("INSERT INTO users (id, username, display_name, password_hash) VALUES (%s,'explain_user','x','x')", (user,))
chats = [uuid.uuid4() for _ in range(5)]
for cid in chats:
conn.execute("INSERT INTO chats (id, title, created_by) VALUES (%s,'t',%s)", (cid, user))
conn.execute("INSERT INTO chat_members (chat_id, user_id) VALUES (%s,%s)", (cid, user))
# Вставка вперемешку по времени (как у живых чатов): created_at — равномерная шкала + микросекунды.
for i, cid in enumerate(chats):
n = TARGET_N if i == 0 else NOISE_N
conn.execute("""INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), %s, %s, 'message ' || g,
timestamptz '2026-01-01 00:00:00+00' + (g * 5 + %s) * interval '1 second' + (random()*1000) * interval '1 microsecond'
FROM generate_series(1, %s) g""", (cid, user, i, n))
conn.execute("VACUUM (ANALYZE) messages")
target = chats[0]
total = conn.execute("SELECT count(*) FROM messages").fetchone()[0]
in_chat = conn.execute("SELECT count(*) FROM messages WHERE chat_id=%s", (target,)).fetchone()[0]
# Курсор «глубоко в истории»: элемент №15000 по (created_at, id).
ts, mid = conn.execute("""SELECT created_at, id FROM messages WHERE chat_id=%s
ORDER BY created_at, id OFFSET %s LIMIT 1""", (target, min(15000, in_chat - PAGE - 1))).fetchone()
def explain(title, sql, params, pre=None, post=None):
print("\n" + "=" * 100 + "\n" + title + "\n" + "=" * 100)
with conn.transaction(force_rollback=True): # DDL в PostgreSQL транзакционен: индекс вернётся
if pre:
conn.execute(pre)
for line in conn.execute("EXPLAIN (ANALYZE, BUFFERS) " + sql, params).fetchall():
print(line[0])
print("rows total=%d, rows in target chat=%d, page=%d" % (total, in_chat, PAGE))
print(conn.execute("SELECT version()").fetchone()[0])
first = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id=%s ORDER BY created_at, id LIMIT %s"
keyset = ("SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id=%s "
"AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s")
offset = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id=%s ORDER BY created_at, id OFFSET 15000 LIMIT %s"
drop = "DROP INDEX messages_chat_created_id_idx"
explain("A. Первая страница, keyset, С индексом (chat_id, created_at, id)", first, (target, PAGE + 1))
explain("B. Страница на глубине ~15 000, keyset, С индексом", keyset, (target, ts, mid, PAGE + 1))
explain("C. Та же страница, keyset, БЕЗ индекса (индекс удалён в откатываемой транзакции)", keyset, (target, ts, mid, PAGE + 1), pre=drop)
explain("D. Та же глубина через OFFSET 15000, С индексом (антипример)", offset, (target, PAGE + 1))
explain("E. Первая страница, БЕЗ индекса", first, (target, PAGE + 1), pre=drop)
print("\nИндексы messages:")
for row in conn.execute("SELECT indexname, indexdef FROM pg_indexes WHERE tablename='messages' ORDER BY 1").fetchall():
print(" ", row[0], "->", row[1])
print("Размер индекса: ", conn.execute("SELECT pg_size_pretty(pg_relation_size('messages_chat_created_id_idx'))").fetchone()[0],
"; таблицы:", conn.execute("SELECT pg_size_pretty(pg_relation_size('messages'))").fetchone()[0])
if __name__ == "__main__":
main()
+17
View File
@@ -0,0 +1,17 @@
#!/bin/sh
# Создаёт локальные секреты (случайные пароли) и .env. Идемпотентно: существующие секреты НЕ перезаписываются
# (иначе пароль в уже инициализированном томе PostgreSQL перестал бы совпадать). Каталог secrets/ в .gitignore.
set -eu
cd "$(dirname "$0")/.."
umask 077
mkdir -p secrets
rand() { if command -v openssl >/dev/null 2>&1; then openssl rand -hex 24; else python3 -c 'import secrets;print(secrets.token_hex(24))'; fi; }
put() { [ -f "secrets/$1" ] || { printf '%s' "$2" > "secrets/$1"; echo "created secrets/$1"; }; }
[ -f secrets/postgres_password ] || printf '%s' "$(rand)" > secrets/postgres_password
[ -f secrets/valkey_password ] || printf '%s' "$(rand)" > secrets/valkey_password
PG=$(cat secrets/postgres_password); VK=$(cat secrets/valkey_password)
put database_url "postgresql://messenger:${PG}@postgres:5432/messenger"
put cache_url "redis://:${VK}@valkey:6379/0"
# Файлы читаются процессами контейнеров с другим UID, поэтому 644; доступ ограничивает каталог secrets/ (700).
chmod 700 secrets && chmod 644 secrets/*
[ -f .env ] || { cp .env.example .env; echo "created .env"; }
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
set -eu
# TODO: copy to restart.sh and implement using YOUR compose/stack services.
# Restart API and ALL used state stores (DB, sessions, S3, broker as applicable).
# Preserve persistent volumes and schema/data; never down -v / flush / re-seed.
# Allow operator unseal if required in lab7; do not print credentials.
# Exit 0 only after the intended action; test waits for API readiness separately.
printf '%s
' 'TODO: implement restart.sh for your infrastructure' >&2
exit 2
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
# Hook для tests/resilience.py --mode persistence: перезапускает API и ВСЕ хранилища (PostgreSQL, Valkey),
# тома сохраняются. Никаких down -v / flush / повторного seed. Вывод скрыт тестом, секреты не печатаются.
set -eu
cd "$(dirname "$0")/.."
DC="docker compose --env-file .env"
$DC stop api valkey postgres # SIGTERM; compose сам останавливает в обратном порядке зависимостей
$DC up -d # поднимает postgres+valkey, повторно гонит idempotent-миграции, затем api
i=0
until [ "$(docker inspect -f '{{.State.Health.Status}}' "$($DC ps -q api)" 2>/dev/null || true)" = "healthy" ]; do
i=$((i + 1))
[ "$i" -le 70 ] || { echo "api did not become healthy" >&2; exit 1; }
sleep 1
done
+21
View File
@@ -0,0 +1,21 @@
#!/bin/sh
# Восстановление dump в НОВОЕ хранилище (временный контейнер со свежим томом) и сверка с рабочей БД.
# Использование: sh scripts/restore-check.sh backups/messenger-XXXX.dump
# ВНИМАНИЕ: не запускался в среде подготовки (там нет Docker); эквивалентный эксперимент выполнен через
# scripts/track2_restore.py на двух отдельных серверах PostgreSQL (см. evidence/track2-restore.json).
set -eu
cd "$(dirname "$0")/.."
DUMP="${1:?usage: restore-check.sh <dump>}"
DC="docker compose --env-file .env"
NAME="pg-restore-check-$$"
trap 'docker rm -f "$NAME" >/dev/null 2>&1 || true' EXIT
docker run -d --name "$NAME" -e POSTGRES_PASSWORD=restore-check-only -e POSTGRES_USER=messenger -e POSTGRES_DB=restored postgres:17.6-alpine >/dev/null
i=0; until docker exec "$NAME" pg_isready -U messenger -d restored >/dev/null 2>&1; do i=$((i+1)); [ $i -le 60 ] || exit 1; sleep 1; done
START=$(date +%s)
docker exec -i "$NAME" pg_restore -U messenger -d restored --no-owner --exit-on-error < "$DUMP"
docker exec "$NAME" psql -U messenger -d restored -qc "ANALYZE"
echo "restore+analyze seconds: $(( $(date +%s) - START ))"
Q="select 'users',count(*) from users union all select 'chats',count(*) from chats union all select 'chat_members',count(*) from chat_members union all select 'messages',count(*) from messages union all select 'messages_md5', md5(string_agg(id::text||chat_id::text||sender_id::text||created_at::text||md5(text), ',' order by id))::text from messages"
echo "--- restored ---"; docker exec "$NAME" psql -U messenger -d restored -At -c "$Q"
echo "--- source (rows after the dump may differ: that is the RPO window) ---"
$DC exec -T postgres psql -U messenger -d messenger -At -c "$Q"
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
# Проверка expiry: пересоздаёт api с SESSION_TTL_SECONDS=2, гоняет tests_student/test_expiry.py, возвращает штатный TTL.
set -eu
cd "$(dirname "$0")/.."
DC="docker compose --env-file .env"
wait_healthy() { i=0; until [ "$(docker inspect -f '{{.State.Health.Status}}' "$($DC ps -q api)")" = "healthy" ]; do i=$((i+1)); [ $i -le 60 ] || exit 1; sleep 1; done; }
trap 'SESSION_TTL_SECONDS= $DC up -d --force-recreate api >/dev/null; wait_healthy' EXIT
SESSION_TTL_SECONDS=2 $DC up -d --force-recreate api; wait_healthy
(cd tests_student && python3 test_expiry.py 2)
+186
View File
@@ -0,0 +1,186 @@
#!/usr/bin/env python3
"""Трек 2 (часть A): план запроса истории на ≥100 000 сообщений с НЕРАВНОМЕРНЫМИ чатами.
Гипотеза H1: составной индекс (chat_id, created_at, id) даёт страницу истории за O(размер страницы)
для любого чата и любой глубины, а индекс только по chat_id или «глобальный» (created_at, id) — нет;
покрывающий вариант (INCLUDE) читает меньше страниц, но платит за это размером индекса и стоимостью записи.
Гипотеза H2: каждый дополнительный индекс заметно удорожает вставку, и цена растёт с шириной индекса.
Запуск (ОТДЕЛЬНАЯ БД эксперимента, не рабочая!):
DATABASE_URL_FILE=/path/to/url python scripts/track2_indexes.py --out evidence/track2-indexes.json
"""
import argparse
import json
import os
import platform
import statistics
import sys
import time
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import psycopg # noqa: E402
from app import config, migrate # noqa: E402
# Неравномерные размеры: 1 гигант, 2 больших, 10 средних, 50 малых, 500 «почти пустых» = ровно 100 000.
SIZES = [50000, 20000, 10000] + [1000] * 10 + [100] * 50 + [10] * 500
CONFIGS = [
("C0_pk_only", None),
("C1_chat_id", "CREATE INDEX hist_idx ON messages (chat_id)"),
("C2_chat_created_id", "CREATE INDEX hist_idx ON messages (chat_id, created_at, id)"),
("C3_covering_include", "CREATE INDEX hist_idx ON messages (chat_id, created_at, id) INCLUDE (sender_id, text)"),
("C4_global_created_id", "CREATE INDEX hist_idx ON messages (created_at, id)"),
]
COLS = "id, chat_id, sender_id, text, created_at"
PAGE = 50
READ_RUNS, WARMUP = 300, 20
SINGLE_INSERTS = 2000
BULK_REPEATS = 3
def pct(values, p):
values = sorted(values)
return values[max(0, min(len(values) - 1, int(round(p * len(values) + 0.5)) - 1))]
def seed_source(conn):
"""Один раз генерируем данные в messages_src: тексты разной длины (10–400 символов), время по возрастанию в чате."""
conn.execute("TRUNCATE messages, chat_members, chats, sessions, users CASCADE")
conn.execute("DROP TABLE IF EXISTS messages_src")
user = uuid.uuid4()
conn.execute("INSERT INTO users (id, username, display_name, password_hash) VALUES (%s,'track2_user','x','x')", (user,))
chats = [uuid.uuid4() for _ in SIZES]
for cid in chats:
conn.execute("INSERT INTO chats (id, title, created_by) VALUES (%s,'t',%s)", (cid, user))
conn.execute("INSERT INTO chat_members (chat_id, user_id) VALUES (%s,%s)", (cid, user))
conn.execute("""CREATE TABLE messages_src AS
SELECT gen_random_uuid() AS id, c.chat_id, %s::uuid AS sender_id,
left(repeat('сообщение ', 60), 10 + (random() * 390)::int) AS text,
timestamptz '2026-01-01 00:00:00+00' + (g.n * 7 + c.idx) * interval '1 second'
+ (random() * 1000) * interval '1 microsecond' AS created_at
FROM (SELECT unnest(%s::uuid[]) AS chat_id, unnest(%s::int[]) AS size, generate_series(1, %s) AS idx) c
CROSS JOIN LATERAL generate_series(1, c.size) AS g(n)""", (user, chats, SIZES, len(SIZES)))
conn.execute("ANALYZE messages_src")
return user, chats
def reset_history_index(conn, ddl):
conn.execute("DROP INDEX IF EXISTS messages_chat_created_id_idx")
conn.execute("DROP INDEX IF EXISTS hist_idx")
if ddl:
conn.execute(ddl)
def measure_writes(conn, ddl, chats):
"""Стоимость записи: (1) пакетная загрузка 100 000 строк, (2) 2000 одиночных INSERT с коммитом (как работает API)."""
bulk = []
for _ in range(BULK_REPEATS):
conn.execute("TRUNCATE messages")
reset_history_index(conn, ddl)
t0 = time.perf_counter()
conn.execute("INSERT INTO messages (id, chat_id, sender_id, text, created_at) SELECT * FROM messages_src")
bulk.append(time.perf_counter() - t0)
conn.execute("VACUUM (ANALYZE) messages")
# Одиночные вставки: чаты выбираются пропорционально размеру (больше сообщений — чаще пишут), autocommit = COMMIT на каждую.
import random
rnd = random.Random(42)
sender = conn.execute("SELECT sender_id FROM messages LIMIT 1").fetchone()[0]
singles = []
for _ in range(SINGLE_INSERTS):
cid = rnd.choices(chats, weights=SIZES)[0]
t0 = time.perf_counter()
conn.execute("INSERT INTO messages (id, chat_id, sender_id, text) VALUES (%s,%s,%s,%s)",
(uuid.uuid4(), cid, sender, "одиночная вставка " + "x" * rnd.randint(10, 300)))
singles.append(time.perf_counter() - t0)
conn.execute("VACUUM (ANALYZE) messages")
size = lambda rel: conn.execute("SELECT pg_relation_size(%s)", (rel,)).fetchone()[0]
return {
"bulk_insert_100k_seconds": {"runs": [round(x, 3) for x in bulk], "median": round(statistics.median(bulk), 3)},
"single_insert_ms": {"n": len(singles), "mean": round(1000 * statistics.mean(singles), 3),
"p50": round(1000 * pct(singles, .50), 3), "p95": round(1000 * pct(singles, .95), 3),
"p99": round(1000 * pct(singles, .99), 3)},
"history_index_bytes": size("hist_idx") if ddl else 0,
"table_bytes": size("messages"),
"pk_index_bytes": size("messages_pkey"),
}
def plan_summary(conn, sql, params):
plan = conn.execute("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + sql, params).fetchone()[0][0]
nodes, stack = [], [plan["Plan"]]
while stack:
node = stack.pop()
nodes.append(node["Node Type"])
stack.extend(node.get("Plans", []))
root = plan["Plan"]
return {"nodes": nodes[::-1], "execution_ms": round(plan["Execution Time"], 3),
"shared_buffers": root.get("Shared Hit Blocks", 0) + root.get("Shared Read Blocks", 0),
"rows": root["Actual Rows"]}
def measure_reads(conn, queries):
out = {}
for name, (sql, params) in queries.items():
for _ in range(WARMUP):
conn.execute(sql, params).fetchall()
times = []
for _ in range(READ_RUNS):
t0 = time.perf_counter()
conn.execute(sql, params).fetchall()
times.append(time.perf_counter() - t0)
out[name] = {"p50_ms": round(1000 * pct(times, .50), 3), "p95_ms": round(1000 * pct(times, .95), 3),
"plan": plan_summary(conn, sql, params)}
return out
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="evidence/track2-indexes.json")
ap.add_argument("--reverse", action="store_true", help="обратный порядок конфигураций: проверка, что результат не зависит от порядка/прогрева")
args = ap.parse_args()
migrate.apply_all(config.DATABASE_URL)
with psycopg.connect(config.DATABASE_URL, autocommit=True) as conn:
user, chats = seed_source(conn)
total = conn.execute("SELECT count(*) FROM messages_src").fetchone()[0]
big, small, mid = chats[0], chats[-1], chats[3] # 50 000 / 10 / 1 000 сообщений
deep_ts, deep_id = conn.execute("SELECT created_at, id FROM messages_src WHERE chat_id=%s ORDER BY created_at, id OFFSET 40000 LIMIT 1", (big,)).fetchone()
mid_ts, mid_id = conn.execute("SELECT created_at, id FROM messages_src WHERE chat_id=%s ORDER BY created_at, id OFFSET 500 LIMIT 1", (mid,)).fetchone()
first = "SELECT %s FROM messages WHERE chat_id=%%s ORDER BY created_at, id LIMIT %%s" % COLS
keyset = "SELECT %s FROM messages WHERE chat_id=%%s AND (created_at, id) > (%%s, %%s) ORDER BY created_at, id LIMIT %%s" % COLS
offset = "SELECT %s FROM messages WHERE chat_id=%%s ORDER BY created_at, id OFFSET 40000 LIMIT %%s" % COLS
queries = {
"Q1_first_page_giant_50k": (first, (big, PAGE + 1)),
"Q2_deep_keyset_giant_at_40k": (keyset, (big, deep_ts, deep_id, PAGE + 1)),
"Q3_first_page_tiny_10": (first, (small, PAGE + 1)),
"Q4_keyset_mid_1k_at_500": (keyset, (mid, mid_ts, mid_id, PAGE + 1)),
"Q5_OFFSET_40000_giant": (offset, (big, PAGE + 1)),
}
env = {
"postgres": conn.execute("SHOW server_version").fetchone()[0],
"shared_buffers": conn.execute("SHOW shared_buffers").fetchone()[0],
"work_mem": conn.execute("SHOW work_mem").fetchone()[0],
"fsync": conn.execute("SHOW fsync").fetchone()[0],
"synchronous_commit": conn.execute("SHOW synchronous_commit").fetchone()[0],
"cpus": os.cpu_count(), "python": platform.python_version(), "platform": platform.platform(),
}
results = {"environment": env, "dataset": {"messages": total, "chats": len(SIZES), "chat_size_distribution":
{"50000": 1, "20000": 1, "10000": 1, "1000": 10, "100": 50, "10": 500}, "text_length_chars": "10..400 (равномерно)",
"page_size": PAGE, "read_runs": READ_RUNS, "warmup": WARMUP, "single_inserts": SINGLE_INSERTS,
"bulk_repeats": BULK_REPEATS, "order": "reverse" if args.reverse else "forward"}, "configs": {}}
for name, ddl in (reversed(CONFIGS) if args.reverse else CONFIGS):
print("== %s ==" % name, flush=True)
w = measure_writes(conn, ddl, chats)
r = measure_reads(conn, queries)
results["configs"][name] = {"ddl": ddl, "writes": w, "reads": r}
print(" bulk %.2fs, single p50 %.3f ms, index %.1f MiB; Q1 p50 %.3f ms, Q2 p50 %.3f ms, Q3 p50 %.3f ms" % (
w["bulk_insert_100k_seconds"]["median"], w["single_insert_ms"]["p50"], w["history_index_bytes"] / 2**20,
r["Q1_first_page_giant_50k"]["p50_ms"], r["Q2_deep_keyset_giant_at_40k"]["p50_ms"], r["Q3_first_page_tiny_10"]["p50_ms"]), flush=True)
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
Path(args.out).write_text(json.dumps(results, ensure_ascii=False, indent=2) + "\n")
print("saved", args.out)
if __name__ == "__main__":
main()
+142
View File
@@ -0,0 +1,142 @@
#!/usr/bin/env python3
"""Трек 2 (часть B): backup -> восстановление в НОВОЕ хранилище -> сверка количества и выбранных сообщений.
Источник — БД эксперимента (после track2_indexes.py, 100 000+ сообщений). Приёмник — другой кластер/сервер PostgreSQL
(новый каталог данных, другой порт), т.е. настоящее «новое хранилище», а не вторая БД в том же кластере.
python scripts/track2_restore.py --src-url URL --dst-admin-url URL(postgres db на новом сервере) \
--dst-db messenger_restored --pg-bin /usr/lib/postgresql/16/bin --dump-file /tmp/t2.dump --out evidence/track2-restore.json
Что измеряется: длительность pg_dump и pg_restore (+ ANALYZE), размер дампа, полнота сверки, окно потери данных (RPO):
записи, сделанные ПОСЛЕ начала дампа, в копии отсутствуют; отозванная после дампа сессия в копии «воскресает».
"""
import argparse
import json
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
from urllib.parse import urlsplit, urlunsplit
import psycopg
def with_db(url, db):
parts = urlsplit(url)
return urlunsplit(parts._replace(path="/" + db))
def facts(conn):
"""Отпечаток состояния: счётчики, md5 содержимого, схема, выбранные строки."""
one = lambda sql, *p: conn.execute(sql, p).fetchone()[0]
out = {"counts": {t: one("SELECT count(*) FROM " + t) for t in ("users", "chats", "chat_members", "messages", "sessions")}}
out["messages_md5"] = one("""SELECT md5(string_agg(id::text || '|' || chat_id::text || '|' || sender_id::text || '|' ||
created_at::text || '|' || md5(text), ',' ORDER BY id)) FROM messages""")
out["per_chat_counts_md5"] = one("SELECT md5(string_agg(chat_id::text || ':' || n, ',' ORDER BY chat_id)) FROM (SELECT chat_id, count(*) n FROM messages GROUP BY chat_id) s")
out["indexes_md5"] = one("SELECT md5(string_agg(indexname || indexdef, E'\\n' ORDER BY indexname)) FROM pg_indexes WHERE schemaname='public'")
out["constraints_md5"] = one("SELECT md5(string_agg(conname || pg_get_constraintdef(oid), E'\\n' ORDER BY conname)) FROM pg_constraint WHERE connamespace = 'public'::regnamespace")
out["migrations"] = [r[0] for r in conn.execute("SELECT version FROM schema_migrations ORDER BY 1").fetchall()]
big = one("SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) DESC, chat_id LIMIT 1")
picks = {
"first_overall": "SELECT * FROM messages ORDER BY created_at, id LIMIT 1",
"last_overall": "SELECT * FROM messages ORDER BY created_at DESC, id DESC LIMIT 1",
"giant_chat_item_15000": "SELECT * FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET 15000 LIMIT 1" % big,
"longest_text": "SELECT * FROM messages ORDER BY char_length(text) DESC, id LIMIT 1",
"shortest_text": "SELECT * FROM messages ORDER BY char_length(text), id LIMIT 1",
"pseudo_random_by_md5": "SELECT * FROM messages ORDER BY md5(id::text) LIMIT 1",
}
out["selected_messages"] = {k: one("SELECT row_to_json(s)::text FROM (%s) s" % q) for k, q in picks.items()}
return out
def run(cmd, env):
t0 = time.perf_counter()
proc = subprocess.run(cmd, env=env, capture_output=True, text=True)
dt = time.perf_counter() - t0
if proc.returncode:
raise SystemExit("command failed (%s): %s" % (cmd[0], proc.stderr[-800:]))
return dt
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--src-url", required=True)
ap.add_argument("--dst-admin-url", required=True, help="URL БД postgres на НОВОМ сервере (создание БД восстановления)")
ap.add_argument("--dst-db", default="messenger_restored")
ap.add_argument("--pg-bin", default="/usr/lib/postgresql/16/bin")
ap.add_argument("--dump-file", default="/tmp/track2.dump")
ap.add_argument("--out", default="evidence/track2-restore.json")
a = ap.parse_args()
dst_url = with_db(a.dst_admin_url, a.dst_db)
env = dict(os.environ)
with psycopg.connect(a.src_url, autocommit=True) as src:
# 1. Приводим источник к «боевой» схеме: без вспомогательной таблицы, канонический индекс истории, одна сессия.
src.execute("DROP TABLE IF EXISTS messages_src")
src.execute("DROP INDEX IF EXISTS hist_idx")
src.execute("CREATE INDEX IF NOT EXISTS messages_chat_created_id_idx ON messages (chat_id, created_at, id)")
user = src.execute("SELECT id FROM users LIMIT 1").fetchone()[0]
token_hash = os.urandom(32)
src.execute("INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s,%s, now() + interval '1 day')", (token_hash, user))
src.execute("VACUUM (ANALYZE)")
before = facts(src)
db_bytes = src.execute("SELECT pg_database_size(current_database())").fetchone()[0]
# 2. Backup: логический, custom-формат; снапшот согласован на момент НАЧАЛА pg_dump.
t_dump = run([a.pg_bin + "/pg_dump", "-Fc", "--no-owner", "-f", a.dump_file, "-d", a.src_url], env)
dump_bytes = Path(a.dump_file).stat().st_size
# 3. Окно потери (RPO): изменения ПОСЛЕ дампа — 500 новых сообщений и отзыв сессии.
chat = src.execute("SELECT chat_id FROM messages LIMIT 1").fetchone()[0]
sender = src.execute("SELECT sender_id FROM messages WHERE chat_id=%s LIMIT 1", (chat,)).fetchone()[0]
for i in range(500):
src.execute("INSERT INTO messages (id, chat_id, sender_id, text) VALUES (%s,%s,%s,%s)", (uuid.uuid4(), chat, sender, "после бэкапа %d" % i))
src.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,))
after_source = facts(src)
# 4. Новое хранилище: другая БД на другом сервере.
with psycopg.connect(a.dst_admin_url, autocommit=True) as admin:
admin.execute('DROP DATABASE IF EXISTS "%s"' % a.dst_db)
admin.execute('CREATE DATABASE "%s"' % a.dst_db)
t_restore = run([a.pg_bin + "/pg_restore", "--no-owner", "--exit-on-error", "-d", dst_url, a.dump_file], env)
with psycopg.connect(dst_url, autocommit=True) as dst:
t0 = time.perf_counter()
dst.execute("ANALYZE") # pg_dump не переносит статистику планировщика: без ANALYZE планы могут быть плохими
t_analyze = time.perf_counter() - t0
restored = facts(dst)
plans = {}
big = dst.execute("SELECT chat_id FROM messages GROUP BY chat_id ORDER BY count(*) DESC, chat_id LIMIT 1").fetchone()[0]
plan = dst.execute("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) SELECT id, chat_id, sender_id, text, created_at FROM messages "
"WHERE chat_id=%s ORDER BY created_at, id LIMIT 51", (big,)).fetchone()[0][0]
node = plan["Plan"]
while node.get("Plans"):
node = node["Plans"][0]
plans["first_page_giant"] = {"node": node["Node Type"], "index": node.get("Index Name"), "execution_ms": round(plan["Execution Time"], 3)}
session_present = dst.execute("SELECT count(*) FROM sessions WHERE token_hash=%s", (token_hash,)).fetchone()[0]
# 5. Сверка: копия обязана совпасть с источником НА МОМЕНТ ДАМПА.
checks = {k: before[k] == restored[k] for k in before}
checks["selected_messages"] = {k: before["selected_messages"][k] == restored["selected_messages"][k] for k in before["selected_messages"]}
ok = all(v if isinstance(v, bool) else all(v.values()) for v in checks.values())
rpo = {
"messages_lost_in_restore": after_source["counts"]["messages"] - restored["counts"]["messages"],
"revoked_session_resurrected_in_restore": bool(session_present),
}
result = {
"snapshot_consistent_with_restore": ok, "checks": checks, "rpo_window_observed": rpo,
"timing_seconds": {"pg_dump": round(t_dump, 2), "pg_restore": round(t_restore, 2), "analyze_after_restore": round(t_analyze, 2),
"rto_dump_free_restore_plus_analyze": round(t_restore + t_analyze, 2)},
"sizes_bytes": {"source_database": db_bytes, "dump_file": dump_bytes},
"counts_at_dump": before["counts"], "counts_restored": restored["counts"], "counts_source_after_extra_writes": after_source["counts"],
"restored_plan": plans,
}
Path(a.out).parent.mkdir(parents=True, exist_ok=True)
Path(a.out).write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(result, ensure_ascii=False, indent=2))
raise SystemExit(0 if ok else 1)
if __name__ == "__main__":
main()
+23
View File
@@ -0,0 +1,23 @@
#!/usr/bin/env python3
"""Full OpenAPI and JSON Schema checks; install requirements-dev.txt first."""
import json
from pathlib import Path
from openapi_spec_validator import validate
from jsonschema import Draft202012Validator
root = Path(__file__).resolve().parents[1]
spec = json.loads((root/'contracts/openapi.json').read_text())
validate(spec)
from referencing import Registry, Resource
# The base URI resolves local #/components references inside examples.
resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource))
for name, example in json.loads((root/'contracts/examples.json').read_text()).items():
schema = {'$ref': 'urn:mtusi:openapi#/components/schemas/' + name}
Draft202012Validator(schema, registry=registry).validate(example)
for path in (root/'contracts').glob('*.schema.json'):
schema = json.loads(path.read_text())
Draft202012Validator.check_schema(schema)
for example in schema.get('examples',[]):
Draft202012Validator(schema).validate(example)
print('PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.')