Files
mtusi2026lab2/bench/track2/backup_restore.py
T
Alexey BodanyukandClaude Opus 5.5 eb92e8e91e bench: трек 2 — планы запросов, стоимость записи, backup/restore со сверкой
plans.py: 121 120 сообщений (60k/15k/2k/80), 3 индекса x keyset/OFFSET x 3 размера
чата, pgbench в отдельном контейнере; WAL на вставку детерминированно.
backup_restore.py: pg_dump живой БД во время записи через API, восстановление в
новый контейнер и volume, сверка MD5 по чатам, API на восстановленной БД.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 20:08:13 +03:00

282 lines
14 KiB
Python

#!/usr/bin/env python3
"""Track 2, part 2: backup -> restore into NEW storage -> reconciliation, with RPO/RTO.
python bench/track2/backup_restore.py --repeats 3 # stand must be running
1. The live stand database gets (once) >= 100 000 synthetic messages with uneven chat sizes.
2. A writer sends messages through the real API (POST /messages) the whole time.
3. `pg_dump -Fc` of the live database (one consistent MVCC snapshot) to ./backups (git-ignored).
4. Restore into NEW storage: a fresh PostgreSQL container with a NEW volume (`pg_restore -j 2`).
5. Reconciliation source vs restored: row counts, per-chat MD5 of every message
(id, created_at, text), the first/last 50 messages of the biggest chat, schema_migrations.
Writer messages: the restored ones must be an exact PREFIX of the confirmed ones
(consistent snapshot, no torn state); the rest is the data-loss window (RPO).
6. The app itself is started against the restored database and reads the writer's chat
with a fresh login.
Results: evidence/track2/backup.json. No passwords or tokens are written.
"""
from __future__ import annotations
import argparse
import base64
import datetime as dt
import json
import secrets
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
import httpx2 as httpx
ROOT = Path(__file__).resolve().parents[2]
IMAGE = "postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c"
APP_IMAGE = "mtusi-messenger:lab2"
RESTORE_PG, RESTORE_API, RESTORE_NET, RESTORE_VOL = "mtusi-restore-pg", "mtusi-restore-api", "mtusi-restore-net", "mtusi-restore-pgdata"
BASE = "http://127.0.0.1:8080"
SEED_USER = "00000000-0000-4000-8000-0000000c0001"
SEED_LIVE = f"""
\\set ON_ERROR_STOP on
BEGIN;
INSERT INTO users (id, username, display_name, password_hash)
VALUES ('{SEED_USER}', 'track2_live', convert_to('Track 2 live', 'UTF8'), 'not-a-login') ON CONFLICT DO NOTHING;
CREATE TEMP TABLE sizes AS
SELECT n, CASE WHEN n = 1 THEN 60000 WHEN n <= 3 THEN 15000 WHEN n <= 11 THEN 2000 ELSE 80 END AS size,
md5('track2-live-chat-' || n)::uuid AS chat_id
FROM generate_series(1::int, 200::int) AS n;
INSERT INTO chats (id, title, created_by, created_at)
SELECT chat_id, convert_to('track2 live ' || n, 'UTF8'), '{SEED_USER}', timestamptz '2026-02-01 00:00:00+00'
FROM sizes ON CONFLICT DO NOTHING;
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
SELECT chat_id, '{SEED_USER}', 0, timestamptz '2026-02-01 00:00:00+00' FROM sizes ON CONFLICT DO NOTHING;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), s.chat_id, '{SEED_USER}',
convert_to(left(repeat('сообщение ', 40), 5 + (random() * 195)::int), 'UTF8'),
timestamptz '2026-02-01 00:00:00+00' + g * interval '1 second'
FROM sizes s CROSS JOIN LATERAL generate_series(1::int, s.size) AS g
WHERE NOT EXISTS (SELECT 1 FROM messages m WHERE m.chat_id = s.chat_id);
UPDATE chats c SET last_message_at = m.mx FROM (SELECT chat_id, max(created_at) mx FROM messages GROUP BY chat_id) m
WHERE m.chat_id = c.id AND c.created_by = '{SEED_USER}';
COMMIT;
"""
RECONCILE_SQL = """
\\pset footer off
SELECT 'count:' || t || ':' || n FROM (
SELECT 'users' t, count(*) n FROM users UNION ALL SELECT 'chats', count(*) FROM chats
UNION ALL SELECT 'chat_members', count(*) FROM chat_members UNION ALL SELECT 'messages', count(*) FROM messages
UNION ALL SELECT 'schema_migrations', count(*) FROM schema_migrations) x ORDER BY t;
SELECT 'chat:' || chat_id || ':' || count(*) || ':' ||
md5(string_agg(id::text || '|' || created_at::text || '|' || encode(text, 'hex'), ',' ORDER BY created_at, id))
FROM messages WHERE chat_id <> '{writer_chat}' GROUP BY chat_id ORDER BY chat_id;
SELECT 'big:' || md5(string_agg(x, ',')) FROM (
(SELECT id::text || '|' || encode(text, 'hex') AS x FROM messages WHERE chat_id = md5('track2-live-chat-1')::uuid
ORDER BY created_at, id LIMIT 50)
UNION ALL
(SELECT * FROM (SELECT id::text || '|' || encode(text, 'hex') FROM messages WHERE chat_id = md5('track2-live-chat-1')::uuid
ORDER BY created_at DESC, id DESC LIMIT 50) y)) z;
SELECT 'mig:' || string_agg(version || '=' || checksum, ',' ORDER BY version) FROM schema_migrations;
SELECT 'idx:' || string_agg(indexname, ',' ORDER BY indexname) FROM pg_indexes WHERE schemaname = 'public';
SELECT 'writer:' || string_agg(id::text, ',' ORDER BY created_at, id) FROM messages WHERE chat_id = '{writer_chat}';
"""
def sh(args, *, input_bytes: bytes | None = None, check: bool = True, timeout: float = 600) -> subprocess.CompletedProcess:
result = subprocess.run(args, input=input_bytes, capture_output=True, timeout=timeout, cwd=ROOT)
if check and result.returncode != 0:
raise RuntimeError(f"{' '.join(map(str, args[:5]))}: {result.stderr.decode(errors='replace')[-1500:]}")
return result
def live_psql(sql: str) -> str:
return sh(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
"-X", "-q", "-At", "-f", "-"], input_bytes=sql.encode()).stdout.decode().strip()
def restored_psql(sql: str) -> str:
return sh(["docker", "exec", "-i", RESTORE_PG, "psql", "-U", "messenger", "-d", "messenger",
"-X", "-q", "-At", "-f", "-"], input_bytes=sql.encode()).stdout.decode().strip()
def parse(text: str) -> dict:
out: dict = {"count": {}, "chat": {}}
for line in text.splitlines():
kind, _, rest = line.partition(":")
if kind == "count":
name, _, value = rest.partition(":")
out["count"][name] = int(value)
elif kind == "chat":
chat, _, digest = rest.partition(":")
out["chat"][chat] = digest
else:
out[kind] = rest
return out
class Writer(threading.Thread):
"""Real API traffic during the backup: one message every ~10 ms, confirmations recorded."""
def __init__(self):
super().__init__(daemon=True)
self.client = httpx.Client(base_url=BASE, timeout=10)
self.username = "writer_" + uuid.uuid4().hex[:10]
self.password = secrets.token_urlsafe(16)
user = self.client.post("/api/v1/users", json={"username": self.username, "display_name": "Writer",
"password": self.password}).json()
auth = base64.b64encode(f"{self.username}:{self.password}".encode()).decode()
token = self.client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + auth}).json()["token"]
self.headers = {"Authorization": "Bearer " + token}
self.chat = self.client.post("/api/v1/chats", headers=self.headers,
json={"title": "writer", "member_ids": [user["id"]]}).json()["id"]
self.confirmed: list[tuple[str, str]] = []
self.errors = 0
self.stop = threading.Event()
def run(self) -> None:
n = 0
while not self.stop.is_set():
r = self.client.post(f"/api/v1/chats/{self.chat}/messages", headers=self.headers, json={"text": f"w{n}"})
if r.status_code == 201:
self.confirmed.append((r.json()["id"], r.json()["created_at"]))
else:
self.errors += 1
n += 1
time.sleep(0.01)
def new_storage() -> str:
for cmd in (["docker", "rm", "-f", "-v", RESTORE_API, RESTORE_PG], ["docker", "volume", "rm", "-f", RESTORE_VOL],
["docker", "network", "rm", RESTORE_NET]):
sh(cmd, check=False)
sh(["docker", "network", "create", RESTORE_NET])
password = secrets.token_urlsafe(18)
sh(["docker", "run", "-d", "--name", RESTORE_PG, "--network", RESTORE_NET, "--cpus", "2", "--memory", "2g",
"-e", f"POSTGRES_PASSWORD={password}", "-e", "POSTGRES_USER=messenger", "-e", "POSTGRES_DB=messenger",
"-v", f"{RESTORE_VOL}:/var/lib/postgresql", IMAGE])
for _ in range(120):
if sh(["docker", "exec", RESTORE_PG, "pg_isready", "-U", "messenger", "-d", "messenger", "-q"], check=False).returncode == 0:
time.sleep(1)
return password
time.sleep(0.5)
raise RuntimeError("restore PostgreSQL did not start")
def app_on_restored(password: str, writer: Writer, expected_ids: list[str]) -> dict:
"""Run our API image against the restored DB (sessions in PostgreSQL) and read the writer chat."""
url = f"postgresql://messenger:{password}@{RESTORE_PG}:5432/messenger"
sh(["docker", "run", "-d", "--name", RESTORE_API, "--network", RESTORE_NET, "-p", "127.0.0.1:18090:8080",
"-e", "SESSION_STORE=postgres", "-e", f"DATABASE_URL={url}", "-e", "DATABASE_URL_FILE=", APP_IMAGE])
client = httpx.Client(base_url="http://127.0.0.1:18090", timeout=10)
for _ in range(60):
try:
if client.get("/health/ready").status_code == 200:
break
except httpx.HTTPError:
pass
time.sleep(0.5)
auth = base64.b64encode(f"{writer.username}:{writer.password}".encode()).decode()
token = client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + auth}).json()["token"]
seen, cursor = [], None
while True:
q = "?limit=100" + (f"&cursor={cursor}" if cursor else "")
page = client.get(f"/api/v1/chats/{writer.chat}/messages{q}", headers={"Authorization": "Bearer " + token}).json()
seen += [m["id"] for m in page["items"]]
cursor = page["next_cursor"]
if not cursor:
break
sh(["docker", "rm", "-f", RESTORE_API], check=False)
return {"login_ok": True, "history_via_api_equals_restored": seen == expected_ids, "messages_read": len(seen)}
def one_round(round_no: int, backups: Path) -> dict:
writer = Writer()
writer.start()
time.sleep(2) # writer is in steady state before the backup starts
dump_file = backups / f"messenger-round{round_no}.dump"
started_wall = dt.datetime.now(dt.timezone.utc)
started = time.monotonic()
with dump_file.open("wb") as f:
proc = subprocess.run(["docker", "compose", "exec", "-T", "postgres", "pg_dump", "-U", "messenger", "-d", "messenger",
"-Fc", "-Z", "6"], stdout=f, stderr=subprocess.PIPE, cwd=ROOT, timeout=600)
if proc.returncode != 0:
raise RuntimeError(proc.stderr.decode()[-1000:])
dump_s = time.monotonic() - started
time.sleep(2) # keep writing after the dump: these confirmed messages are not in the backup
writer.stop.set()
writer.join()
source = parse(live_psql(RECONCILE_SQL.format(writer_chat=writer.chat)))
password = new_storage()
started = time.monotonic()
sh(["docker", "exec", "-i", RESTORE_PG, "sh", "-c", "cat > /tmp/backup.dump"], input_bytes=dump_file.read_bytes())
sh(["docker", "exec", RESTORE_PG, "pg_restore", "-U", "messenger", "-d", "messenger", "--no-owner", "-j", "2",
"--exit-on-error", "/tmp/backup.dump"])
restore_s = time.monotonic() - started
started = time.monotonic()
restored_psql("ANALYZE;")
analyze_s = time.monotonic() - started
restored = parse(restored_psql(RECONCILE_SQL.format(writer_chat=writer.chat)))
confirmed_ids = [m[0] for m in writer.confirmed]
restored_writer = restored.get("writer", "").split(",") if restored.get("writer") else []
prefix_ok = restored_writer == confirmed_ids[:len(restored_writer)]
lost = confirmed_ids[len(restored_writer):]
last_in_backup = writer.confirmed[len(restored_writer) - 1][1] if restored_writer else None
first_lost = writer.confirmed[len(restored_writer)][1] if lost else None
other_counts_equal = {k: source["count"][k] == restored["count"][k] for k in source["count"] if k != "messages"}
api_check = app_on_restored(password, writer, restored_writer)
result = {
"round": round_no, "backup_started_utc": started_wall.isoformat(timespec="seconds"),
"dump_seconds": round(dump_s, 2), "dump_bytes": dump_file.stat().st_size,
"restore_seconds": round(restore_s, 2), "analyze_seconds": round(analyze_s, 2),
"rto_seconds_restore_plus_analyze": round(restore_s + analyze_s, 2),
"source_counts_after_writer_stopped": source["count"], "restored_counts": restored["count"],
"messages_in_restored": restored["count"]["messages"],
"per_chat_md5_equal": source["chat"] == restored["chat"], "chats_compared": len(source["chat"]),
"big_chat_first_last_50_equal": source["big"] == restored["big"],
"schema_migrations_equal": source["mig"] == restored["mig"], "indexes_equal": source["idx"] == restored["idx"],
"other_table_counts_equal": other_counts_equal,
"writer": {"confirmed": len(confirmed_ids), "errors": writer.errors, "in_backup": len(restored_writer),
"restored_is_exact_prefix": prefix_ok, "lost_confirmed_after_snapshot": len(lost),
"last_message_in_backup_at": last_in_backup, "first_lost_message_at": first_lost},
"app_on_restored_db": api_check,
}
result["reconciled"] = bool(result["per_chat_md5_equal"] and result["big_chat_first_last_50_equal"]
and result["schema_migrations_equal"] and result["indexes_equal"]
and all(other_counts_equal.values()) and prefix_ok
and api_check["history_via_api_equals_restored"])
return result
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--repeats", type=int, default=3)
parser.add_argument("--out", default=str(ROOT / "evidence/track2/backup.json"))
args = parser.parse_args()
backups = ROOT / "backups"
backups.mkdir(exist_ok=True)
live_psql(SEED_LIVE)
rounds = []
try:
for i in range(1, args.repeats + 1):
r = one_round(i, backups)
rounds.append(r)
print(json.dumps({k: r[k] for k in ("round", "dump_seconds", "dump_bytes", "restore_seconds", "messages_in_restored",
"reconciled")} | {"writer": r["writer"]}, ensure_ascii=False), flush=True)
finally:
for cmd in (["docker", "rm", "-f", "-v", RESTORE_API, RESTORE_PG], ["docker", "volume", "rm", "-f", RESTORE_VOL],
["docker", "network", "rm", RESTORE_NET]):
sh(cmd, check=False)
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
Path(args.out).write_text(json.dumps({"image": IMAGE, "rounds": rounds}, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8", newline="\n")
print("done:", args.out)
return 0 if all(r["reconciled"] for r in rounds) else 1
if __name__ == "__main__":
sys.exit(main())