From d978b8ff4d32f5d5d8f230bf1b48047e8a6490a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Sat, 7 Nov 2026 08:12:09 +0300 Subject: [PATCH] research: compare keyset indexes and restore an isolated database --- evidence/public-persistence.txt | 8 + research/query_plan.py | 442 ++++++++++++++++++++++++++++++++ 2 files changed, 450 insertions(+) create mode 100644 evidence/public-persistence.txt create mode 100644 research/query_plan.py diff --git a/evidence/public-persistence.txt b/evidence/public-persistence.txt new file mode 100644 index 00000000..c8b8303c --- /dev/null +++ b/evidence/public-persistence.txt @@ -0,0 +1,8 @@ +Date: 2026-11-07 +Runtime source: 2f0a8672be67af9e62d2693c3df9bd683fcdae57 +Command: make test-persistence ACTION_SCRIPT=scripts/restart.sh + +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +Actual API/PostgreSQL/Valkey restart evidence: lifecycle.json diff --git a/research/query_plan.py b/research/query_plan.py new file mode 100644 index 00000000..2066f3af --- /dev/null +++ b/research/query_plan.py @@ -0,0 +1,442 @@ +"""Изолированная БД: планы, цена записи, backup/restore без рабочих данных.""" + +import argparse +import hashlib +import json +import resource +import time +from datetime import UTC, datetime, timedelta +from pathlib import Path +from uuid import UUID, uuid4, uuid5 + +import psycopg +from psycopg import sql +from psycopg.conninfo import make_conninfo + +from src.settings import secret + +NAMESPACE = UUID("2e52c91b-b28f-4990-988c-7e8e15c9f131") +USER = uuid5(NAMESPACE, "research-user") +ORIGIN = datetime(2026, 1, 1, tzinfo=UTC) +SIZES = [90_000, 10_000, 1000, 100, 10] + + +def connect(database): + return psycopg.connect( + make_conninfo(secret("DATABASE_URL"), dbname=database), + autocommit=True, + connect_timeout=3, + prepare_threshold=None, + options="-c statement_timeout=60000 -c timezone=UTC", + ) + + +def safe_database(name): + import re + + if not re.fullmatch(r"(?:research|restored)_[a-f0-9]{12}", name): + raise ValueError("Refusing database outside research scope") + + +def snapshot(conn): + counts = {} + for table in [ + "users", + "chats", + "members", + "messages", + "sessions", + "schema_migrations", + ]: + counts[table] = conn.execute( + sql.SQL("SELECT count(*) FROM {}").format(sql.Identifier(table)) + ).fetchone()[0] + checksum = hashlib.sha256() + with conn.transaction(), conn.cursor(name="snapshot_messages") as cursor: + cursor.execute( + "SELECT id,chat_id,sender_id,text,created_at " + "FROM messages ORDER BY id" + ) + for row in cursor: + checksum.update( + json.dumps( + [str(value) for value in row], separators=(",", ":") + ).encode() + ) + checksum.update(b"\n") + selected = [] + for index, size in enumerate(SIZES): + chat = uuid5(NAMESPACE, "chat:" + str(index)) + for offset in [0, size // 2, size - 1]: + row = conn.execute( + "SELECT id,chat_id,sender_id,text,created_at " + "FROM messages WHERE chat_id=%s " + "ORDER BY created_at,id OFFSET %s LIMIT 1", + (chat, offset), + ).fetchone() + selected.append([str(value) for value in row]) + return { + "counts": counts, + "messages_sha256": checksum.hexdigest(), + "selected_messages": selected, + } + + +def inside_plan(database): + safe_database(database) + with connect("messenger") as admin: + admin.execute( + sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database)) + ) + with connect(database) as conn: + migration = Path("migrations/001_core.sql").read_bytes() + conn.execute(migration.decode()) + conn.execute( + "CREATE TABLE schema_migrations (name text PRIMARY KEY, " + "checksum text NOT NULL, " + "applied_at timestamptz NOT NULL DEFAULT now())" + ) + conn.execute( + "INSERT INTO schema_migrations(name,checksum) VALUES(%s,%s)", + ("001_core.sql", hashlib.sha256(migration).hexdigest()), + ) + conn.execute( + "INSERT INTO users VALUES(%s,%s,%s,%s,%s)", + (USER, "research_user", "Synthetic", "not-a-login", ORIGIN), + ) + cursors = [] + for index, size in enumerate(SIZES): + chat = uuid5(NAMESPACE, "chat:" + str(index)) + conn.execute( + "INSERT INTO chats VALUES(%s,%s,%s)", + (chat, "Research " + str(index), ORIGIN), + ) + conn.execute("INSERT INTO members VALUES(%s,%s,0)", (chat, USER)) + conn.execute( + "INSERT INTO messages SELECT " + "md5(%s || ':' || n::text)::uuid,%s,%s," + "rpad('message ' || n,80,'x'),%s + n*interval '1 microsecond' " + "FROM generate_series(1,%s) n", + (str(chat), chat, USER, ORIGIN, size), + ) + for position, offset in [ + ("head", 0), + ("middle", size // 2), + ("tail", max(0, size - 52)), + ]: + row = conn.execute( + "SELECT created_at,id FROM messages WHERE chat_id=%s " + "ORDER BY created_at,id OFFSET %s LIMIT 1", + (chat, offset), + ).fetchone() + cursors.append((size, position, chat, row[0], row[1])) + assert ( + conn.execute("SELECT count(*) FROM messages").fetchone()[0] + >= 100_000 + ) + query = ( + "SELECT id,chat_id,sender_id,text,created_at FROM messages " + "WHERE chat_id=%s AND (created_at,id)>(%s,%s) " + "ORDER BY created_at,id LIMIT 51" + ) + reads, writes, indexes = [], [], [] + for repeat, order in enumerate( + [ + ["chat_only", "keyset"], + ["keyset", "chat_only"], + ["chat_only", "keyset"], + ], + 1, + ): + for variant in order: + conn.execute("DROP INDEX IF EXISTS messages_history_idx") + conn.execute("DROP INDEX IF EXISTS messages_chat_only_idx") + if variant == "chat_only": + conn.execute( + "CREATE INDEX messages_chat_only_idx " + "ON messages(chat_id)" + ) + index_name = "messages_chat_only_idx" + else: + conn.execute( + "CREATE INDEX messages_history_idx " + "ON messages(chat_id,created_at,id)" + ) + index_name = "messages_history_idx" + conn.execute("VACUUM (ANALYZE) messages") + index_bytes = conn.execute( + "SELECT pg_relation_size(%s)", (index_name,) + ).fetchone()[0] + indexes.append( + { + "repeat": repeat, + "variant": variant, + "bytes": index_bytes, + } + ) + for size, position, chat, at, mid in cursors: + parameters = (chat, at, mid) + conn.execute(query, parameters).fetchall() + plan = conn.execute( + "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + query, + parameters, + ).fetchone()[0][0] + reads.append( + { + "repeat": repeat, + "variant": variant, + "chat_messages": size, + "position": position, + "execution_ms": plan["Execution Time"], + "planning_ms": plan["Planning Time"], + "shared_hits": plan["Plan"].get( + "Shared Hit Blocks", 0 + ), + "shared_reads": plan["Plan"].get( + "Shared Read Blocks", 0 + ), + "plan": plan, + } + ) + # Окно записи включает COMMIT; cleanup за границей окна. + batch = "write:" + uuid4().hex + chat = uuid5(NAMESPACE, "chat:0") + lsn_before = conn.execute( + "SELECT pg_current_wal_insert_lsn()" + ).fetchone()[0] + started = time.perf_counter() + with conn.transaction(): + conn.execute( + "INSERT INTO messages SELECT " + "md5(%s || ':' || n::text)::uuid,%s,%s," + "rpad('write ' || n,80,'x'),%s + " + "n*interval '1 microsecond' " + "FROM generate_series(1,1000) n", + (batch, chat, USER, ORIGIN + timedelta(days=1)), + ) + elapsed = time.perf_counter() - started + wal = conn.execute( + "SELECT pg_wal_lsn_diff(pg_current_wal_insert_lsn(),%s)", + (lsn_before,), + ).fetchone()[0] + writes.append( + { + "repeat": repeat, + "variant": variant, + "inserted": 1000, + "insert_and_commit_seconds": elapsed, + "wal_bytes": int(wal), + } + ) + conn.execute( + "DELETE FROM messages WHERE id IN " + "(SELECT md5(%s || ':' || n::text)::uuid " + "FROM generate_series(1,1000) n)", + (batch,), + ) + assert conn.execute( + "SELECT count(*) FROM messages" + ).fetchone()[0] == sum(SIZES) + result = { + "dataset": { + "chat_sizes": SIZES, + "messages": sum(SIZES), + "text_characters": 80, + "source": ( + "synthetic SQL seed; no production API " + "or working tables modified" + ), + }, + "reads": reads, + "writes": writes, + "indexes": indexes, + "snapshot_before_backup": snapshot(conn), + "research_worker_maxrss_kib": resource.getrusage( + resource.RUSAGE_SELF + ).ru_maxrss, + } + return result + + +def inside_after(database): + safe_database(database) + with connect(database) as conn: + value = uuid4() + conn.execute( + "INSERT INTO messages VALUES(%s,%s,%s,%s,%s)", + ( + value, + uuid5(NAMESPACE, "chat:0"), + USER, + "Committed after backup snapshot", + ORIGIN + timedelta(days=2), + ), + ) + return { + "post_backup_message": str(value), + "count": conn.execute("SELECT count(*) FROM messages").fetchone()[ + 0 + ], + } + + +def inside_snapshot(database): + safe_database(database) + with connect(database) as conn: + return snapshot(conn) + + +def inside_cleanup(databases): + with connect("messenger") as admin: + for database in databases: + safe_database(database) + admin.execute( + sql.SQL("DROP DATABASE IF EXISTS {} WITH (FORCE)").format( + sql.Identifier(database) + ) + ) + return {"scoped_databases_removed": databases} + + +def outside(output): + from scripts.stand import ROOT, command, compose + + if output.exists(): + raise SystemExit("Choose a new output file") + identifier = uuid4().hex[:12] + database, restored = "research_" + identifier, "restored_" + identifier + dump_path = "/tmp/" + database + ".dump" + mount = str(ROOT / "research") + ":/app/research:ro" + + def inside(phase, selected): + return json.loads( + compose( + "run", + "--rm", + "--no-deps", + "-v", + mount, + "api", + "python", + "-m", + "research.query_plan", + "--phase", + phase, + "--database", + selected, + ) + ) + + result = { + "date": datetime.now(UTC).isoformat(), + "commit": command(["git", "rev-parse", "HEAD"]), + "command": "python -m research.query_plan", + "db_cpu_limit": 1, + "db_memory_limit_bytes": 536870912, + "prepared_statements": "disabled for comparable custom EXPLAIN plans", + } + try: + result.update(inside("plan", database)) + started = time.perf_counter() + compose( + "exec", + "-T", + "postgres", + "pg_dump", + "-U", + "messenger", + "-Fc", + "-d", + database, + "-f", + dump_path, + ) + result["backup_seconds"] = time.perf_counter() - started + result["backup_bytes"] = int( + compose("exec", "-T", "postgres", "stat", "-c", "%s", dump_path) + ) + after = inside("after", database) + started = time.perf_counter() + compose( + "exec", "-T", "postgres", "createdb", "-U", "messenger", restored + ) + compose( + "exec", + "-T", + "postgres", + "pg_restore", + "-U", + "messenger", + "--exit-on-error", + "-d", + restored, + dump_path, + ) + result["restore_seconds"] = time.perf_counter() - started + checked = inside("snapshot", restored) + expected = result["snapshot_before_backup"] + assert expected == checked + assert after["count"] == expected["counts"]["messages"] + 1 + result["restore_verification"] = { + "snapshot_equal": True, + "selected_messages_checked": len(checked["selected_messages"]), + "restored": checked, + "post_snapshot_commits_not_restored": 1, + "source_count_after_backup": after["count"], + "post_snapshot_message": after["post_backup_message"], + } + result["postgres_cgroup_memory_current"] = int( + compose( + "exec", + "-T", + "postgres", + "cat", + "/sys/fs/cgroup/memory.current", + ) + ) + result["postgres_cgroup_memory_peak"] = int( + compose( + "exec", "-T", "postgres", "cat", "/sys/fs/cgroup/memory.peak" + ) + ) + result["result"] = "PASS" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(result, ensure_ascii=False, indent=2) + "\n" + ) + print( + "PASS:", + result["dataset"]["messages"], + "messages;", + len(result["reads"]), + "EXPLAIN plans; backup/restore verified", + ) + finally: + # Удаляются только две БД с UUID именно этого запуска. + for selected in [database, restored]: + inside("cleanup", selected) + compose("exec", "-T", "postgres", "rm", "-f", dump_path) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path) + parser.add_argument( + "--phase", choices=["plan", "after", "snapshot", "cleanup"] + ) + parser.add_argument("--database") + args = parser.parse_args() + if args.phase: + function = { + "plan": inside_plan, + "after": inside_after, + "snapshot": inside_snapshot, + "cleanup": lambda name: inside_cleanup([name]), + }[args.phase] + print(json.dumps(function(args.database), ensure_ascii=False)) + else: + outside(args.output) + + +if __name__ == "__main__": + main()