Files
mtusi2026lab2/research/worker.py
T

184 lines
7.8 KiB
Python

"""Execute inside the API container, where psycopg and scoped DB credentials exist."""
import hashlib
import json
import os
import sys
import time
from pathlib import Path
import psycopg
from psycopg import sql
from psycopg.conninfo import make_conninfo
from psycopg.rows import dict_row
MODE, DATABASE, RESTORED = sys.argv[1:]
if not DATABASE.startswith("research_") or not RESTORED.startswith("restored_"):
raise SystemExit("Only isolated research databases are supported")
DSN = Path(os.environ["DATABASE_URL_FILE"]).read_text().strip()
def connect(database="postgres"):
return psycopg.connect(
make_conninfo(DSN, dbname=database), autocommit=True, row_factory=dict_row
)
def fingerprint(conn):
counts = {
table: conn.execute(
sql.SQL("SELECT count(*) AS n FROM {}").format(sql.Identifier(table))
).fetchone()["n"]
for table in ("users", "chats", "members", "messages", "sessions")
}
selected = conn.execute(
"SELECT id::text,chat_id::text,sender_id::text,text,"
"to_char(created_at AT TIME ZONE 'UTC',"
'\'YYYY-MM-DD"T"HH24:MI:SS.US"Z"\') AS created_at '
"FROM messages ORDER BY id LIMIT 20"
).fetchall()
digest = conn.execute(
"SELECT md5(string_agg(id::text || ':' || chat_id::text || ':' || "
"sender_id::text || ':' || text || ':' || "
"extract(epoch FROM created_at)::text, '|' ORDER BY id)) AS checksum "
"FROM messages"
).fetchone()["checksum"]
return {"counts": counts, "selected_messages": selected, "all_messages_md5": digest}
def measure():
with connect() as admin:
admin.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(DATABASE)))
with connect(DATABASE) as conn:
schema = Path("migrations/001_initial.sql").read_text()
conn.execute(schema)
conn.execute(
"INSERT INTO users VALUES (md5('research-user')::uuid,'research_user',"
"'Синтетический исследователь','not-a-login-hash','2026-01-01T00:00:00Z')"
)
conn.execute(
"INSERT INTO chats SELECT md5('chat-'||g)::uuid,'research chat '||g,"
"'2026-01-01T00:00:00Z'::timestamptz FROM generate_series(1,10) g"
)
conn.execute("INSERT INTO members SELECT id,md5('research-user')::uuid,0 FROM chats")
seed_start = time.perf_counter()
conn.execute(
"INSERT INTO messages SELECT md5('message-'||g)::uuid,"
"md5('chat-' || CASE WHEN g<=80000 THEN 1 WHEN g<=95000 THEN 2 "
"ELSE 3+(g%8) END)::uuid,md5('research-user')::uuid,"
"'synthetic message '||g,'2026-01-01T00:00:00Z'::timestamptz + g*interval '1 microsecond' "
"FROM generate_series(1,100000) g"
)
seed_seconds = time.perf_counter() - seed_start
conn.execute("VACUUM ANALYZE messages")
cursor = conn.execute(
"SELECT created_at,id FROM messages "
"WHERE chat_id=md5('chat-1')::uuid "
"ORDER BY created_at,id OFFSET 60000 LIMIT 1"
).fetchone()
query = (
"SELECT id,chat_id,sender_id,text,created_at FROM messages "
"WHERE chat_id=md5('chat-1')::uuid AND (created_at,id)>(%s,%s) "
"ORDER BY created_at,id LIMIT 51"
)
params = (cursor["created_at"], cursor["id"])
phases = []
for repeat, order in enumerate(
(("chat_only", "composite"), ("composite", "chat_only"), ("chat_only", "composite")),
start=1,
):
for variant in order:
conn.execute("DROP INDEX IF EXISTS messages_history_idx")
conn.execute("DROP INDEX IF EXISTS research_history_idx")
columns = "chat_id" if variant == "chat_only" else "chat_id,created_at,id"
conn.execute("CREATE INDEX research_history_idx ON messages (" + columns + ")")
conn.execute("VACUUM ANALYZE messages")
for _ in range(5):
conn.execute(query, params).fetchall()
trials = []
for _ in range(7):
plan = conn.execute(
"EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + query, params
).fetchone()["QUERY PLAN"][0]
trials.append(plan)
size = conn.execute("SELECT pg_relation_size('research_history_idx') AS bytes")
index_bytes = size.fetchone()["bytes"]
before = conn.execute("SELECT pg_current_wal_insert_lsn() AS lsn").fetchone()["lsn"]
write_start = time.perf_counter()
conn.execute(
"INSERT INTO messages SELECT md5('write-'||g)::uuid,"
"md5('chat-1')::uuid,md5('research-user')::uuid,"
"'synthetic write '||g,'2026-01-02T00:00:00Z'::timestamptz + g*interval '1 microsecond' "
"FROM generate_series(1,5000) g"
)
write_seconds = time.perf_counter() - write_start
wal_bytes = conn.execute(
"SELECT pg_wal_lsn_diff(pg_current_wal_insert_lsn(),%s) AS n", (before,)
).fetchone()["n"]
conn.execute(
"DELETE FROM messages WHERE created_at>='2026-01-02T00:00:00Z'::timestamptz"
)
phases.append(
{
"repeat": repeat,
"variant": variant,
"plans": trials,
"index_bytes_before_write": index_bytes,
"insert_rows": 5000,
"insert_commit_seconds": write_seconds,
"insert_wal_bytes": int(wal_bytes),
}
)
conn.execute("DROP INDEX research_history_idx")
conn.execute("CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id)")
conn.execute("VACUUM ANALYZE messages")
distribution = conn.execute(
"SELECT c.title,count(*) AS messages FROM messages m "
"JOIN chats c ON c.id=m.chat_id GROUP BY c.title ORDER BY messages DESC"
).fetchall()
settings = conn.execute(
"SELECT name,setting FROM pg_settings WHERE name IN "
"('server_version','shared_buffers','fsync','synchronous_commit',"
"'full_page_writes','max_connections','work_mem') ORDER BY name"
).fetchall()
return {
"seed_seconds": seed_seconds,
"rows": 100000,
"schema_sha256": hashlib.sha256(schema.encode()).hexdigest(),
"distribution": distribution,
"query": query,
"cursor_position": 60000,
"settings": settings,
"phases": phases,
"source": fingerprint(conn),
}
def main():
if MODE == "measure":
result = measure()
elif MODE == "create_restore":
with connect() as admin:
admin.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(RESTORED)))
result = {"created": RESTORED}
elif MODE == "verify":
with connect(DATABASE) as original, connect(RESTORED) as restored:
a, b = fingerprint(original), fingerprint(restored)
if a != b:
raise RuntimeError("Restored data mismatch")
result = {"equal": True, "restored": b}
elif MODE == "cleanup":
with connect() as admin:
for name in (DATABASE, RESTORED):
admin.execute(
sql.SQL("DROP DATABASE IF EXISTS {} WITH (FORCE)").format(sql.Identifier(name))
)
result = {"removed_only": [DATABASE, RESTORED]}
else:
raise ValueError("Unknown research operation")
print(json.dumps(result, ensure_ascii=False, default=str))
if __name__ == "__main__":
main()