443 lines
15 KiB
Python
443 lines
15 KiB
Python
"""Изолированная БД: планы, цена записи, backup/restore без рабочих данных."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import resource
|
|
import time
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from uuid import UUID, uuid4, uuid5
|
|
|
|
import psycopg
|
|
from psycopg import sql
|
|
from psycopg.conninfo import make_conninfo
|
|
|
|
from src.settings import secret
|
|
|
|
NAMESPACE = UUID("2e52c91b-b28f-4990-988c-7e8e15c9f131")
|
|
USER = uuid5(NAMESPACE, "research-user")
|
|
ORIGIN = datetime(2026, 1, 1, tzinfo=UTC)
|
|
SIZES = [90_000, 10_000, 1000, 100, 10]
|
|
|
|
|
|
def connect(database):
|
|
return psycopg.connect(
|
|
make_conninfo(secret("DATABASE_URL"), dbname=database),
|
|
autocommit=True,
|
|
connect_timeout=3,
|
|
prepare_threshold=None,
|
|
options="-c statement_timeout=60000 -c timezone=UTC",
|
|
)
|
|
|
|
|
|
def safe_database(name):
|
|
import re
|
|
|
|
if not re.fullmatch(r"(?:research|restored)_[a-f0-9]{12}", name):
|
|
raise ValueError("Refusing database outside research scope")
|
|
|
|
|
|
def snapshot(conn):
|
|
counts = {}
|
|
for table in [
|
|
"users",
|
|
"chats",
|
|
"members",
|
|
"messages",
|
|
"sessions",
|
|
"schema_migrations",
|
|
]:
|
|
counts[table] = conn.execute(
|
|
sql.SQL("SELECT count(*) FROM {}").format(sql.Identifier(table))
|
|
).fetchone()[0]
|
|
checksum = hashlib.sha256()
|
|
with conn.transaction(), conn.cursor(name="snapshot_messages") as cursor:
|
|
cursor.execute(
|
|
"SELECT id,chat_id,sender_id,text,created_at "
|
|
"FROM messages ORDER BY id"
|
|
)
|
|
for row in cursor:
|
|
checksum.update(
|
|
json.dumps(
|
|
[str(value) for value in row], separators=(",", ":")
|
|
).encode()
|
|
)
|
|
checksum.update(b"\n")
|
|
selected = []
|
|
for index, size in enumerate(SIZES):
|
|
chat = uuid5(NAMESPACE, "chat:" + str(index))
|
|
for offset in [0, size // 2, size - 1]:
|
|
row = conn.execute(
|
|
"SELECT id,chat_id,sender_id,text,created_at "
|
|
"FROM messages WHERE chat_id=%s "
|
|
"ORDER BY created_at,id OFFSET %s LIMIT 1",
|
|
(chat, offset),
|
|
).fetchone()
|
|
selected.append([str(value) for value in row])
|
|
return {
|
|
"counts": counts,
|
|
"messages_sha256": checksum.hexdigest(),
|
|
"selected_messages": selected,
|
|
}
|
|
|
|
|
|
def inside_plan(database):
|
|
safe_database(database)
|
|
with connect("messenger") as admin:
|
|
admin.execute(
|
|
sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database))
|
|
)
|
|
with connect(database) as conn:
|
|
migration = Path("migrations/001_core.sql").read_bytes()
|
|
conn.execute(migration.decode())
|
|
conn.execute(
|
|
"CREATE TABLE schema_migrations (name text PRIMARY KEY, "
|
|
"checksum text NOT NULL, "
|
|
"applied_at timestamptz NOT NULL DEFAULT now())"
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO schema_migrations(name,checksum) VALUES(%s,%s)",
|
|
("001_core.sql", hashlib.sha256(migration).hexdigest()),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO users VALUES(%s,%s,%s,%s,%s)",
|
|
(USER, "research_user", "Synthetic", "not-a-login", ORIGIN),
|
|
)
|
|
cursors = []
|
|
for index, size in enumerate(SIZES):
|
|
chat = uuid5(NAMESPACE, "chat:" + str(index))
|
|
conn.execute(
|
|
"INSERT INTO chats VALUES(%s,%s,%s)",
|
|
(chat, "Research " + str(index), ORIGIN),
|
|
)
|
|
conn.execute("INSERT INTO members VALUES(%s,%s,0)", (chat, USER))
|
|
conn.execute(
|
|
"INSERT INTO messages SELECT "
|
|
"md5(%s || ':' || n::text)::uuid,%s,%s,"
|
|
"rpad('message ' || n,80,'x'),%s + n*interval '1 microsecond' "
|
|
"FROM generate_series(1,%s) n",
|
|
(str(chat), chat, USER, ORIGIN, size),
|
|
)
|
|
for position, offset in [
|
|
("head", 0),
|
|
("middle", size // 2),
|
|
("tail", max(0, size - 52)),
|
|
]:
|
|
row = conn.execute(
|
|
"SELECT created_at,id FROM messages WHERE chat_id=%s "
|
|
"ORDER BY created_at,id OFFSET %s LIMIT 1",
|
|
(chat, offset),
|
|
).fetchone()
|
|
cursors.append((size, position, chat, row[0], row[1]))
|
|
assert (
|
|
conn.execute("SELECT count(*) FROM messages").fetchone()[0]
|
|
>= 100_000
|
|
)
|
|
query = (
|
|
"SELECT id,chat_id,sender_id,text,created_at FROM messages "
|
|
"WHERE chat_id=%s AND (created_at,id)>(%s,%s) "
|
|
"ORDER BY created_at,id LIMIT 51"
|
|
)
|
|
reads, writes, indexes = [], [], []
|
|
for repeat, order in enumerate(
|
|
[
|
|
["chat_only", "keyset"],
|
|
["keyset", "chat_only"],
|
|
["chat_only", "keyset"],
|
|
],
|
|
1,
|
|
):
|
|
for variant in order:
|
|
conn.execute("DROP INDEX IF EXISTS messages_history_idx")
|
|
conn.execute("DROP INDEX IF EXISTS messages_chat_only_idx")
|
|
if variant == "chat_only":
|
|
conn.execute(
|
|
"CREATE INDEX messages_chat_only_idx "
|
|
"ON messages(chat_id)"
|
|
)
|
|
index_name = "messages_chat_only_idx"
|
|
else:
|
|
conn.execute(
|
|
"CREATE INDEX messages_history_idx "
|
|
"ON messages(chat_id,created_at,id)"
|
|
)
|
|
index_name = "messages_history_idx"
|
|
conn.execute("VACUUM (ANALYZE) messages")
|
|
index_bytes = conn.execute(
|
|
"SELECT pg_relation_size(%s)", (index_name,)
|
|
).fetchone()[0]
|
|
indexes.append(
|
|
{
|
|
"repeat": repeat,
|
|
"variant": variant,
|
|
"bytes": index_bytes,
|
|
}
|
|
)
|
|
for size, position, chat, at, mid in cursors:
|
|
parameters = (chat, at, mid)
|
|
conn.execute(query, parameters).fetchall()
|
|
plan = conn.execute(
|
|
"EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + query,
|
|
parameters,
|
|
).fetchone()[0][0]
|
|
reads.append(
|
|
{
|
|
"repeat": repeat,
|
|
"variant": variant,
|
|
"chat_messages": size,
|
|
"position": position,
|
|
"execution_ms": plan["Execution Time"],
|
|
"planning_ms": plan["Planning Time"],
|
|
"shared_hits": plan["Plan"].get(
|
|
"Shared Hit Blocks", 0
|
|
),
|
|
"shared_reads": plan["Plan"].get(
|
|
"Shared Read Blocks", 0
|
|
),
|
|
"plan": plan,
|
|
}
|
|
)
|
|
# Окно записи включает COMMIT; cleanup за границей окна.
|
|
batch = "write:" + uuid4().hex
|
|
chat = uuid5(NAMESPACE, "chat:0")
|
|
lsn_before = conn.execute(
|
|
"SELECT pg_current_wal_insert_lsn()"
|
|
).fetchone()[0]
|
|
started = time.perf_counter()
|
|
with conn.transaction():
|
|
conn.execute(
|
|
"INSERT INTO messages SELECT "
|
|
"md5(%s || ':' || n::text)::uuid,%s,%s,"
|
|
"rpad('write ' || n,80,'x'),%s + "
|
|
"n*interval '1 microsecond' "
|
|
"FROM generate_series(1,1000) n",
|
|
(batch, chat, USER, ORIGIN + timedelta(days=1)),
|
|
)
|
|
elapsed = time.perf_counter() - started
|
|
wal = conn.execute(
|
|
"SELECT pg_wal_lsn_diff(pg_current_wal_insert_lsn(),%s)",
|
|
(lsn_before,),
|
|
).fetchone()[0]
|
|
writes.append(
|
|
{
|
|
"repeat": repeat,
|
|
"variant": variant,
|
|
"inserted": 1000,
|
|
"insert_and_commit_seconds": elapsed,
|
|
"wal_bytes": int(wal),
|
|
}
|
|
)
|
|
conn.execute(
|
|
"DELETE FROM messages WHERE id IN "
|
|
"(SELECT md5(%s || ':' || n::text)::uuid "
|
|
"FROM generate_series(1,1000) n)",
|
|
(batch,),
|
|
)
|
|
assert conn.execute(
|
|
"SELECT count(*) FROM messages"
|
|
).fetchone()[0] == sum(SIZES)
|
|
result = {
|
|
"dataset": {
|
|
"chat_sizes": SIZES,
|
|
"messages": sum(SIZES),
|
|
"text_characters": 80,
|
|
"source": (
|
|
"synthetic SQL seed; no production API "
|
|
"or working tables modified"
|
|
),
|
|
},
|
|
"reads": reads,
|
|
"writes": writes,
|
|
"indexes": indexes,
|
|
"snapshot_before_backup": snapshot(conn),
|
|
"research_worker_maxrss_kib": resource.getrusage(
|
|
resource.RUSAGE_SELF
|
|
).ru_maxrss,
|
|
}
|
|
return result
|
|
|
|
|
|
def inside_after(database):
|
|
safe_database(database)
|
|
with connect(database) as conn:
|
|
value = uuid4()
|
|
conn.execute(
|
|
"INSERT INTO messages VALUES(%s,%s,%s,%s,%s)",
|
|
(
|
|
value,
|
|
uuid5(NAMESPACE, "chat:0"),
|
|
USER,
|
|
"Committed after backup snapshot",
|
|
ORIGIN + timedelta(days=2),
|
|
),
|
|
)
|
|
return {
|
|
"post_backup_message": str(value),
|
|
"count": conn.execute("SELECT count(*) FROM messages").fetchone()[
|
|
0
|
|
],
|
|
}
|
|
|
|
|
|
def inside_snapshot(database):
|
|
safe_database(database)
|
|
with connect(database) as conn:
|
|
return snapshot(conn)
|
|
|
|
|
|
def inside_cleanup(databases):
|
|
with connect("messenger") as admin:
|
|
for database in databases:
|
|
safe_database(database)
|
|
admin.execute(
|
|
sql.SQL("DROP DATABASE IF EXISTS {} WITH (FORCE)").format(
|
|
sql.Identifier(database)
|
|
)
|
|
)
|
|
return {"scoped_databases_removed": databases}
|
|
|
|
|
|
def outside(output):
|
|
from scripts.stand import ROOT, command, compose
|
|
|
|
if output.exists():
|
|
raise SystemExit("Choose a new output file")
|
|
identifier = uuid4().hex[:12]
|
|
database, restored = "research_" + identifier, "restored_" + identifier
|
|
dump_path = "/tmp/" + database + ".dump"
|
|
mount = str(ROOT / "research") + ":/app/research:ro"
|
|
|
|
def inside(phase, selected):
|
|
return json.loads(
|
|
compose(
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"-v",
|
|
mount,
|
|
"api",
|
|
"python",
|
|
"-m",
|
|
"research.query_plan",
|
|
"--phase",
|
|
phase,
|
|
"--database",
|
|
selected,
|
|
)
|
|
)
|
|
|
|
result = {
|
|
"date": datetime.now(UTC).isoformat(),
|
|
"commit": command(["git", "rev-parse", "HEAD"]),
|
|
"command": "python -m research.query_plan",
|
|
"db_cpu_limit": 1,
|
|
"db_memory_limit_bytes": 536870912,
|
|
"prepared_statements": "disabled for comparable custom EXPLAIN plans",
|
|
}
|
|
try:
|
|
result.update(inside("plan", database))
|
|
started = time.perf_counter()
|
|
compose(
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"pg_dump",
|
|
"-U",
|
|
"messenger",
|
|
"-Fc",
|
|
"-d",
|
|
database,
|
|
"-f",
|
|
dump_path,
|
|
)
|
|
result["backup_seconds"] = time.perf_counter() - started
|
|
result["backup_bytes"] = int(
|
|
compose("exec", "-T", "postgres", "stat", "-c", "%s", dump_path)
|
|
)
|
|
after = inside("after", database)
|
|
started = time.perf_counter()
|
|
compose(
|
|
"exec", "-T", "postgres", "createdb", "-U", "messenger", restored
|
|
)
|
|
compose(
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"pg_restore",
|
|
"-U",
|
|
"messenger",
|
|
"--exit-on-error",
|
|
"-d",
|
|
restored,
|
|
dump_path,
|
|
)
|
|
result["restore_seconds"] = time.perf_counter() - started
|
|
checked = inside("snapshot", restored)
|
|
expected = result["snapshot_before_backup"]
|
|
assert expected == checked
|
|
assert after["count"] == expected["counts"]["messages"] + 1
|
|
result["restore_verification"] = {
|
|
"snapshot_equal": True,
|
|
"selected_messages_checked": len(checked["selected_messages"]),
|
|
"restored": checked,
|
|
"post_snapshot_commits_not_restored": 1,
|
|
"source_count_after_backup": after["count"],
|
|
"post_snapshot_message": after["post_backup_message"],
|
|
}
|
|
result["postgres_cgroup_memory_current"] = int(
|
|
compose(
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"cat",
|
|
"/sys/fs/cgroup/memory.current",
|
|
)
|
|
)
|
|
result["postgres_cgroup_memory_peak"] = int(
|
|
compose(
|
|
"exec", "-T", "postgres", "cat", "/sys/fs/cgroup/memory.peak"
|
|
)
|
|
)
|
|
result["result"] = "PASS"
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
output.write_text(
|
|
json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
print(
|
|
"PASS:",
|
|
result["dataset"]["messages"],
|
|
"messages;",
|
|
len(result["reads"]),
|
|
"EXPLAIN plans; backup/restore verified",
|
|
)
|
|
finally:
|
|
# Удаляются только две БД с UUID именно этого запуска.
|
|
for selected in [database, restored]:
|
|
inside("cleanup", selected)
|
|
compose("exec", "-T", "postgres", "rm", "-f", dump_path)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--output", type=Path)
|
|
parser.add_argument(
|
|
"--phase", choices=["plan", "after", "snapshot", "cleanup"]
|
|
)
|
|
parser.add_argument("--database")
|
|
args = parser.parse_args()
|
|
if args.phase:
|
|
function = {
|
|
"plan": inside_plan,
|
|
"after": inside_after,
|
|
"snapshot": inside_snapshot,
|
|
"cleanup": lambda name: inside_cleanup([name]),
|
|
}[args.phase]
|
|
print(json.dumps(function(args.database), ensure_ascii=False))
|
|
else:
|
|
outside(args.output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|