184 lines
7.8 KiB
Python
184 lines
7.8 KiB
Python
"""Execute inside the API container, where psycopg and scoped DB credentials exist."""
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import psycopg
|
|
from psycopg import sql
|
|
from psycopg.conninfo import make_conninfo
|
|
from psycopg.rows import dict_row
|
|
|
|
MODE, DATABASE, RESTORED = sys.argv[1:]
|
|
if not DATABASE.startswith("research_") or not RESTORED.startswith("restored_"):
|
|
raise SystemExit("Only isolated research databases are supported")
|
|
DSN = Path(os.environ["DATABASE_URL_FILE"]).read_text().strip()
|
|
|
|
|
|
def connect(database="postgres"):
|
|
return psycopg.connect(
|
|
make_conninfo(DSN, dbname=database), autocommit=True, row_factory=dict_row
|
|
)
|
|
|
|
|
|
def fingerprint(conn):
|
|
counts = {
|
|
table: conn.execute(
|
|
sql.SQL("SELECT count(*) AS n FROM {}").format(sql.Identifier(table))
|
|
).fetchone()["n"]
|
|
for table in ("users", "chats", "members", "messages", "sessions")
|
|
}
|
|
selected = conn.execute(
|
|
"SELECT id::text,chat_id::text,sender_id::text,text,"
|
|
"to_char(created_at AT TIME ZONE 'UTC',"
|
|
'\'YYYY-MM-DD"T"HH24:MI:SS.US"Z"\') AS created_at '
|
|
"FROM messages ORDER BY id LIMIT 20"
|
|
).fetchall()
|
|
digest = conn.execute(
|
|
"SELECT md5(string_agg(id::text || ':' || chat_id::text || ':' || "
|
|
"sender_id::text || ':' || text || ':' || "
|
|
"extract(epoch FROM created_at)::text, '|' ORDER BY id)) AS checksum "
|
|
"FROM messages"
|
|
).fetchone()["checksum"]
|
|
return {"counts": counts, "selected_messages": selected, "all_messages_md5": digest}
|
|
|
|
|
|
def measure():
|
|
with connect() as admin:
|
|
admin.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(DATABASE)))
|
|
with connect(DATABASE) as conn:
|
|
schema = Path("migrations/001_initial.sql").read_text()
|
|
conn.execute(schema)
|
|
conn.execute(
|
|
"INSERT INTO users VALUES (md5('research-user')::uuid,'research_user',"
|
|
"'Синтетический исследователь','not-a-login-hash','2026-01-01T00:00:00Z')"
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO chats SELECT md5('chat-'||g)::uuid,'research chat '||g,"
|
|
"'2026-01-01T00:00:00Z'::timestamptz FROM generate_series(1,10) g"
|
|
)
|
|
conn.execute("INSERT INTO members SELECT id,md5('research-user')::uuid,0 FROM chats")
|
|
seed_start = time.perf_counter()
|
|
conn.execute(
|
|
"INSERT INTO messages SELECT md5('message-'||g)::uuid,"
|
|
"md5('chat-' || CASE WHEN g<=80000 THEN 1 WHEN g<=95000 THEN 2 "
|
|
"ELSE 3+(g%8) END)::uuid,md5('research-user')::uuid,"
|
|
"'synthetic message '||g,'2026-01-01T00:00:00Z'::timestamptz + g*interval '1 microsecond' "
|
|
"FROM generate_series(1,100000) g"
|
|
)
|
|
seed_seconds = time.perf_counter() - seed_start
|
|
conn.execute("VACUUM ANALYZE messages")
|
|
cursor = conn.execute(
|
|
"SELECT created_at,id FROM messages "
|
|
"WHERE chat_id=md5('chat-1')::uuid "
|
|
"ORDER BY created_at,id OFFSET 60000 LIMIT 1"
|
|
).fetchone()
|
|
query = (
|
|
"SELECT id,chat_id,sender_id,text,created_at FROM messages "
|
|
"WHERE chat_id=md5('chat-1')::uuid AND (created_at,id)>(%s,%s) "
|
|
"ORDER BY created_at,id LIMIT 51"
|
|
)
|
|
params = (cursor["created_at"], cursor["id"])
|
|
phases = []
|
|
for repeat, order in enumerate(
|
|
(("chat_only", "composite"), ("composite", "chat_only"), ("chat_only", "composite")),
|
|
start=1,
|
|
):
|
|
for variant in order:
|
|
conn.execute("DROP INDEX IF EXISTS messages_history_idx")
|
|
conn.execute("DROP INDEX IF EXISTS research_history_idx")
|
|
columns = "chat_id" if variant == "chat_only" else "chat_id,created_at,id"
|
|
conn.execute("CREATE INDEX research_history_idx ON messages (" + columns + ")")
|
|
conn.execute("VACUUM ANALYZE messages")
|
|
for _ in range(5):
|
|
conn.execute(query, params).fetchall()
|
|
trials = []
|
|
for _ in range(7):
|
|
plan = conn.execute(
|
|
"EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + query, params
|
|
).fetchone()["QUERY PLAN"][0]
|
|
trials.append(plan)
|
|
size = conn.execute("SELECT pg_relation_size('research_history_idx') AS bytes")
|
|
index_bytes = size.fetchone()["bytes"]
|
|
before = conn.execute("SELECT pg_current_wal_insert_lsn() AS lsn").fetchone()["lsn"]
|
|
write_start = time.perf_counter()
|
|
conn.execute(
|
|
"INSERT INTO messages SELECT md5('write-'||g)::uuid,"
|
|
"md5('chat-1')::uuid,md5('research-user')::uuid,"
|
|
"'synthetic write '||g,'2026-01-02T00:00:00Z'::timestamptz + g*interval '1 microsecond' "
|
|
"FROM generate_series(1,5000) g"
|
|
)
|
|
write_seconds = time.perf_counter() - write_start
|
|
wal_bytes = conn.execute(
|
|
"SELECT pg_wal_lsn_diff(pg_current_wal_insert_lsn(),%s) AS n", (before,)
|
|
).fetchone()["n"]
|
|
conn.execute(
|
|
"DELETE FROM messages WHERE created_at>='2026-01-02T00:00:00Z'::timestamptz"
|
|
)
|
|
phases.append(
|
|
{
|
|
"repeat": repeat,
|
|
"variant": variant,
|
|
"plans": trials,
|
|
"index_bytes_before_write": index_bytes,
|
|
"insert_rows": 5000,
|
|
"insert_commit_seconds": write_seconds,
|
|
"insert_wal_bytes": int(wal_bytes),
|
|
}
|
|
)
|
|
conn.execute("DROP INDEX research_history_idx")
|
|
conn.execute("CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id)")
|
|
conn.execute("VACUUM ANALYZE messages")
|
|
distribution = conn.execute(
|
|
"SELECT c.title,count(*) AS messages FROM messages m "
|
|
"JOIN chats c ON c.id=m.chat_id GROUP BY c.title ORDER BY messages DESC"
|
|
).fetchall()
|
|
settings = conn.execute(
|
|
"SELECT name,setting FROM pg_settings WHERE name IN "
|
|
"('server_version','shared_buffers','fsync','synchronous_commit',"
|
|
"'full_page_writes','max_connections','work_mem') ORDER BY name"
|
|
).fetchall()
|
|
return {
|
|
"seed_seconds": seed_seconds,
|
|
"rows": 100000,
|
|
"schema_sha256": hashlib.sha256(schema.encode()).hexdigest(),
|
|
"distribution": distribution,
|
|
"query": query,
|
|
"cursor_position": 60000,
|
|
"settings": settings,
|
|
"phases": phases,
|
|
"source": fingerprint(conn),
|
|
}
|
|
|
|
|
|
def main():
|
|
if MODE == "measure":
|
|
result = measure()
|
|
elif MODE == "create_restore":
|
|
with connect() as admin:
|
|
admin.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(RESTORED)))
|
|
result = {"created": RESTORED}
|
|
elif MODE == "verify":
|
|
with connect(DATABASE) as original, connect(RESTORED) as restored:
|
|
a, b = fingerprint(original), fingerprint(restored)
|
|
if a != b:
|
|
raise RuntimeError("Restored data mismatch")
|
|
result = {"equal": True, "restored": b}
|
|
elif MODE == "cleanup":
|
|
with connect() as admin:
|
|
for name in (DATABASE, RESTORED):
|
|
admin.execute(
|
|
sql.SQL("DROP DATABASE IF EXISTS {} WITH (FORCE)").format(sql.Identifier(name))
|
|
)
|
|
result = {"removed_only": [DATABASE, RESTORED]}
|
|
else:
|
|
raise ValueError("Unknown research operation")
|
|
print(json.dumps(result, ensure_ascii=False, default=str))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|