forked from mtusi2026/mtusi2026lab2
plans.py: 121 120 сообщений (60k/15k/2k/80), 3 индекса x keyset/OFFSET x 3 размера чата, pgbench в отдельном контейнере; WAL на вставку детерминированно. backup_restore.py: pg_dump живой БД во время записи через API, восстановление в новый контейнер и volume, сверка MD5 по чатам, API на восстановленной БД. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
282 lines
14 KiB
Python
282 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Track 2, part 2: backup -> restore into NEW storage -> reconciliation, with RPO/RTO.
|
|
|
|
python bench/track2/backup_restore.py --repeats 3 # stand must be running
|
|
|
|
1. The live stand database gets (once) >= 100 000 synthetic messages with uneven chat sizes.
|
|
2. A writer sends messages through the real API (POST /messages) the whole time.
|
|
3. `pg_dump -Fc` of the live database (one consistent MVCC snapshot) to ./backups (git-ignored).
|
|
4. Restore into NEW storage: a fresh PostgreSQL container with a NEW volume (`pg_restore -j 2`).
|
|
5. Reconciliation source vs restored: row counts, per-chat MD5 of every message
|
|
(id, created_at, text), the first/last 50 messages of the biggest chat, schema_migrations.
|
|
Writer messages: the restored ones must be an exact PREFIX of the confirmed ones
|
|
(consistent snapshot, no torn state); the rest is the data-loss window (RPO).
|
|
6. The app itself is started against the restored database and reads the writer's chat
|
|
with a fresh login.
|
|
Results: evidence/track2/backup.json. No passwords or tokens are written.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import datetime as dt
|
|
import json
|
|
import secrets
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
import httpx2 as httpx
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
IMAGE = "postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c"
|
|
APP_IMAGE = "mtusi-messenger:lab2"
|
|
RESTORE_PG, RESTORE_API, RESTORE_NET, RESTORE_VOL = "mtusi-restore-pg", "mtusi-restore-api", "mtusi-restore-net", "mtusi-restore-pgdata"
|
|
BASE = "http://127.0.0.1:8080"
|
|
SEED_USER = "00000000-0000-4000-8000-0000000c0001"
|
|
|
|
SEED_LIVE = f"""
|
|
\\set ON_ERROR_STOP on
|
|
BEGIN;
|
|
INSERT INTO users (id, username, display_name, password_hash)
|
|
VALUES ('{SEED_USER}', 'track2_live', convert_to('Track 2 live', 'UTF8'), 'not-a-login') ON CONFLICT DO NOTHING;
|
|
CREATE TEMP TABLE sizes AS
|
|
SELECT n, CASE WHEN n = 1 THEN 60000 WHEN n <= 3 THEN 15000 WHEN n <= 11 THEN 2000 ELSE 80 END AS size,
|
|
md5('track2-live-chat-' || n)::uuid AS chat_id
|
|
FROM generate_series(1::int, 200::int) AS n;
|
|
INSERT INTO chats (id, title, created_by, created_at)
|
|
SELECT chat_id, convert_to('track2 live ' || n, 'UTF8'), '{SEED_USER}', timestamptz '2026-02-01 00:00:00+00'
|
|
FROM sizes ON CONFLICT DO NOTHING;
|
|
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
|
|
SELECT chat_id, '{SEED_USER}', 0, timestamptz '2026-02-01 00:00:00+00' FROM sizes ON CONFLICT DO NOTHING;
|
|
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
|
SELECT gen_random_uuid(), s.chat_id, '{SEED_USER}',
|
|
convert_to(left(repeat('сообщение ', 40), 5 + (random() * 195)::int), 'UTF8'),
|
|
timestamptz '2026-02-01 00:00:00+00' + g * interval '1 second'
|
|
FROM sizes s CROSS JOIN LATERAL generate_series(1::int, s.size) AS g
|
|
WHERE NOT EXISTS (SELECT 1 FROM messages m WHERE m.chat_id = s.chat_id);
|
|
UPDATE chats c SET last_message_at = m.mx FROM (SELECT chat_id, max(created_at) mx FROM messages GROUP BY chat_id) m
|
|
WHERE m.chat_id = c.id AND c.created_by = '{SEED_USER}';
|
|
COMMIT;
|
|
"""
|
|
|
|
RECONCILE_SQL = """
|
|
\\pset footer off
|
|
SELECT 'count:' || t || ':' || n FROM (
|
|
SELECT 'users' t, count(*) n FROM users UNION ALL SELECT 'chats', count(*) FROM chats
|
|
UNION ALL SELECT 'chat_members', count(*) FROM chat_members UNION ALL SELECT 'messages', count(*) FROM messages
|
|
UNION ALL SELECT 'schema_migrations', count(*) FROM schema_migrations) x ORDER BY t;
|
|
SELECT 'chat:' || chat_id || ':' || count(*) || ':' ||
|
|
md5(string_agg(id::text || '|' || created_at::text || '|' || encode(text, 'hex'), ',' ORDER BY created_at, id))
|
|
FROM messages WHERE chat_id <> '{writer_chat}' GROUP BY chat_id ORDER BY chat_id;
|
|
SELECT 'big:' || md5(string_agg(x, ',')) FROM (
|
|
(SELECT id::text || '|' || encode(text, 'hex') AS x FROM messages WHERE chat_id = md5('track2-live-chat-1')::uuid
|
|
ORDER BY created_at, id LIMIT 50)
|
|
UNION ALL
|
|
(SELECT * FROM (SELECT id::text || '|' || encode(text, 'hex') FROM messages WHERE chat_id = md5('track2-live-chat-1')::uuid
|
|
ORDER BY created_at DESC, id DESC LIMIT 50) y)) z;
|
|
SELECT 'mig:' || string_agg(version || '=' || checksum, ',' ORDER BY version) FROM schema_migrations;
|
|
SELECT 'idx:' || string_agg(indexname, ',' ORDER BY indexname) FROM pg_indexes WHERE schemaname = 'public';
|
|
SELECT 'writer:' || string_agg(id::text, ',' ORDER BY created_at, id) FROM messages WHERE chat_id = '{writer_chat}';
|
|
"""
|
|
|
|
|
|
def sh(args, *, input_bytes: bytes | None = None, check: bool = True, timeout: float = 600) -> subprocess.CompletedProcess:
|
|
result = subprocess.run(args, input=input_bytes, capture_output=True, timeout=timeout, cwd=ROOT)
|
|
if check and result.returncode != 0:
|
|
raise RuntimeError(f"{' '.join(map(str, args[:5]))}: {result.stderr.decode(errors='replace')[-1500:]}")
|
|
return result
|
|
|
|
|
|
def live_psql(sql: str) -> str:
|
|
return sh(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
|
"-X", "-q", "-At", "-f", "-"], input_bytes=sql.encode()).stdout.decode().strip()
|
|
|
|
|
|
def restored_psql(sql: str) -> str:
|
|
return sh(["docker", "exec", "-i", RESTORE_PG, "psql", "-U", "messenger", "-d", "messenger",
|
|
"-X", "-q", "-At", "-f", "-"], input_bytes=sql.encode()).stdout.decode().strip()
|
|
|
|
|
|
def parse(text: str) -> dict:
|
|
out: dict = {"count": {}, "chat": {}}
|
|
for line in text.splitlines():
|
|
kind, _, rest = line.partition(":")
|
|
if kind == "count":
|
|
name, _, value = rest.partition(":")
|
|
out["count"][name] = int(value)
|
|
elif kind == "chat":
|
|
chat, _, digest = rest.partition(":")
|
|
out["chat"][chat] = digest
|
|
else:
|
|
out[kind] = rest
|
|
return out
|
|
|
|
|
|
class Writer(threading.Thread):
|
|
"""Real API traffic during the backup: one message every ~10 ms, confirmations recorded."""
|
|
|
|
def __init__(self):
|
|
super().__init__(daemon=True)
|
|
self.client = httpx.Client(base_url=BASE, timeout=10)
|
|
self.username = "writer_" + uuid.uuid4().hex[:10]
|
|
self.password = secrets.token_urlsafe(16)
|
|
user = self.client.post("/api/v1/users", json={"username": self.username, "display_name": "Writer",
|
|
"password": self.password}).json()
|
|
auth = base64.b64encode(f"{self.username}:{self.password}".encode()).decode()
|
|
token = self.client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + auth}).json()["token"]
|
|
self.headers = {"Authorization": "Bearer " + token}
|
|
self.chat = self.client.post("/api/v1/chats", headers=self.headers,
|
|
json={"title": "writer", "member_ids": [user["id"]]}).json()["id"]
|
|
self.confirmed: list[tuple[str, str]] = []
|
|
self.errors = 0
|
|
self.stop = threading.Event()
|
|
|
|
def run(self) -> None:
|
|
n = 0
|
|
while not self.stop.is_set():
|
|
r = self.client.post(f"/api/v1/chats/{self.chat}/messages", headers=self.headers, json={"text": f"w{n}"})
|
|
if r.status_code == 201:
|
|
self.confirmed.append((r.json()["id"], r.json()["created_at"]))
|
|
else:
|
|
self.errors += 1
|
|
n += 1
|
|
time.sleep(0.01)
|
|
|
|
|
|
def new_storage() -> str:
|
|
for cmd in (["docker", "rm", "-f", "-v", RESTORE_API, RESTORE_PG], ["docker", "volume", "rm", "-f", RESTORE_VOL],
|
|
["docker", "network", "rm", RESTORE_NET]):
|
|
sh(cmd, check=False)
|
|
sh(["docker", "network", "create", RESTORE_NET])
|
|
password = secrets.token_urlsafe(18)
|
|
sh(["docker", "run", "-d", "--name", RESTORE_PG, "--network", RESTORE_NET, "--cpus", "2", "--memory", "2g",
|
|
"-e", f"POSTGRES_PASSWORD={password}", "-e", "POSTGRES_USER=messenger", "-e", "POSTGRES_DB=messenger",
|
|
"-v", f"{RESTORE_VOL}:/var/lib/postgresql", IMAGE])
|
|
for _ in range(120):
|
|
if sh(["docker", "exec", RESTORE_PG, "pg_isready", "-U", "messenger", "-d", "messenger", "-q"], check=False).returncode == 0:
|
|
time.sleep(1)
|
|
return password
|
|
time.sleep(0.5)
|
|
raise RuntimeError("restore PostgreSQL did not start")
|
|
|
|
|
|
def app_on_restored(password: str, writer: Writer, expected_ids: list[str]) -> dict:
|
|
"""Run our API image against the restored DB (sessions in PostgreSQL) and read the writer chat."""
|
|
url = f"postgresql://messenger:{password}@{RESTORE_PG}:5432/messenger"
|
|
sh(["docker", "run", "-d", "--name", RESTORE_API, "--network", RESTORE_NET, "-p", "127.0.0.1:18090:8080",
|
|
"-e", "SESSION_STORE=postgres", "-e", f"DATABASE_URL={url}", "-e", "DATABASE_URL_FILE=", APP_IMAGE])
|
|
client = httpx.Client(base_url="http://127.0.0.1:18090", timeout=10)
|
|
for _ in range(60):
|
|
try:
|
|
if client.get("/health/ready").status_code == 200:
|
|
break
|
|
except httpx.HTTPError:
|
|
pass
|
|
time.sleep(0.5)
|
|
auth = base64.b64encode(f"{writer.username}:{writer.password}".encode()).decode()
|
|
token = client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + auth}).json()["token"]
|
|
seen, cursor = [], None
|
|
while True:
|
|
q = "?limit=100" + (f"&cursor={cursor}" if cursor else "")
|
|
page = client.get(f"/api/v1/chats/{writer.chat}/messages{q}", headers={"Authorization": "Bearer " + token}).json()
|
|
seen += [m["id"] for m in page["items"]]
|
|
cursor = page["next_cursor"]
|
|
if not cursor:
|
|
break
|
|
sh(["docker", "rm", "-f", RESTORE_API], check=False)
|
|
return {"login_ok": True, "history_via_api_equals_restored": seen == expected_ids, "messages_read": len(seen)}
|
|
|
|
|
|
def one_round(round_no: int, backups: Path) -> dict:
|
|
writer = Writer()
|
|
writer.start()
|
|
time.sleep(2) # writer is in steady state before the backup starts
|
|
dump_file = backups / f"messenger-round{round_no}.dump"
|
|
started_wall = dt.datetime.now(dt.timezone.utc)
|
|
started = time.monotonic()
|
|
with dump_file.open("wb") as f:
|
|
proc = subprocess.run(["docker", "compose", "exec", "-T", "postgres", "pg_dump", "-U", "messenger", "-d", "messenger",
|
|
"-Fc", "-Z", "6"], stdout=f, stderr=subprocess.PIPE, cwd=ROOT, timeout=600)
|
|
if proc.returncode != 0:
|
|
raise RuntimeError(proc.stderr.decode()[-1000:])
|
|
dump_s = time.monotonic() - started
|
|
time.sleep(2) # keep writing after the dump: these confirmed messages are not in the backup
|
|
writer.stop.set()
|
|
writer.join()
|
|
source = parse(live_psql(RECONCILE_SQL.format(writer_chat=writer.chat)))
|
|
|
|
password = new_storage()
|
|
started = time.monotonic()
|
|
sh(["docker", "exec", "-i", RESTORE_PG, "sh", "-c", "cat > /tmp/backup.dump"], input_bytes=dump_file.read_bytes())
|
|
sh(["docker", "exec", RESTORE_PG, "pg_restore", "-U", "messenger", "-d", "messenger", "--no-owner", "-j", "2",
|
|
"--exit-on-error", "/tmp/backup.dump"])
|
|
restore_s = time.monotonic() - started
|
|
started = time.monotonic()
|
|
restored_psql("ANALYZE;")
|
|
analyze_s = time.monotonic() - started
|
|
restored = parse(restored_psql(RECONCILE_SQL.format(writer_chat=writer.chat)))
|
|
|
|
confirmed_ids = [m[0] for m in writer.confirmed]
|
|
restored_writer = restored.get("writer", "").split(",") if restored.get("writer") else []
|
|
prefix_ok = restored_writer == confirmed_ids[:len(restored_writer)]
|
|
lost = confirmed_ids[len(restored_writer):]
|
|
last_in_backup = writer.confirmed[len(restored_writer) - 1][1] if restored_writer else None
|
|
first_lost = writer.confirmed[len(restored_writer)][1] if lost else None
|
|
other_counts_equal = {k: source["count"][k] == restored["count"][k] for k in source["count"] if k != "messages"}
|
|
api_check = app_on_restored(password, writer, restored_writer)
|
|
result = {
|
|
"round": round_no, "backup_started_utc": started_wall.isoformat(timespec="seconds"),
|
|
"dump_seconds": round(dump_s, 2), "dump_bytes": dump_file.stat().st_size,
|
|
"restore_seconds": round(restore_s, 2), "analyze_seconds": round(analyze_s, 2),
|
|
"rto_seconds_restore_plus_analyze": round(restore_s + analyze_s, 2),
|
|
"source_counts_after_writer_stopped": source["count"], "restored_counts": restored["count"],
|
|
"messages_in_restored": restored["count"]["messages"],
|
|
"per_chat_md5_equal": source["chat"] == restored["chat"], "chats_compared": len(source["chat"]),
|
|
"big_chat_first_last_50_equal": source["big"] == restored["big"],
|
|
"schema_migrations_equal": source["mig"] == restored["mig"], "indexes_equal": source["idx"] == restored["idx"],
|
|
"other_table_counts_equal": other_counts_equal,
|
|
"writer": {"confirmed": len(confirmed_ids), "errors": writer.errors, "in_backup": len(restored_writer),
|
|
"restored_is_exact_prefix": prefix_ok, "lost_confirmed_after_snapshot": len(lost),
|
|
"last_message_in_backup_at": last_in_backup, "first_lost_message_at": first_lost},
|
|
"app_on_restored_db": api_check,
|
|
}
|
|
result["reconciled"] = bool(result["per_chat_md5_equal"] and result["big_chat_first_last_50_equal"]
|
|
and result["schema_migrations_equal"] and result["indexes_equal"]
|
|
and all(other_counts_equal.values()) and prefix_ok
|
|
and api_check["history_via_api_equals_restored"])
|
|
return result
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
parser.add_argument("--repeats", type=int, default=3)
|
|
parser.add_argument("--out", default=str(ROOT / "evidence/track2/backup.json"))
|
|
args = parser.parse_args()
|
|
backups = ROOT / "backups"
|
|
backups.mkdir(exist_ok=True)
|
|
live_psql(SEED_LIVE)
|
|
rounds = []
|
|
try:
|
|
for i in range(1, args.repeats + 1):
|
|
r = one_round(i, backups)
|
|
rounds.append(r)
|
|
print(json.dumps({k: r[k] for k in ("round", "dump_seconds", "dump_bytes", "restore_seconds", "messages_in_restored",
|
|
"reconciled")} | {"writer": r["writer"]}, ensure_ascii=False), flush=True)
|
|
finally:
|
|
for cmd in (["docker", "rm", "-f", "-v", RESTORE_API, RESTORE_PG], ["docker", "volume", "rm", "-f", RESTORE_VOL],
|
|
["docker", "network", "rm", RESTORE_NET]):
|
|
sh(cmd, check=False)
|
|
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(args.out).write_text(json.dumps({"image": IMAGE, "rounds": rounds}, ensure_ascii=False, indent=2) + "\n",
|
|
encoding="utf-8", newline="\n")
|
|
print("done:", args.out)
|
|
return 0 if all(r["reconciled"] for r in rounds) else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|