110 lines
8.2 KiB
Python
110 lines
8.2 KiB
Python
"""120k skewed messages: query plans/write cost + real fresh-volume restore.
|
|
|
|
Host orchestrates Docker; --inside runs synthetic SQL in an isolated database.
|
|
No production table is changed and no credentials are printed.
|
|
"""
|
|
import hashlib
|
|
from datetime import datetime
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
def inside(dbname):
|
|
import psycopg
|
|
with psycopg.connect(os.environ['DATABASE_URL'],autocommit=True) as admin:
|
|
admin.execute(psycopg.sql.SQL('CREATE DATABASE {}').format(psycopg.sql.Identifier(dbname)))
|
|
with psycopg.connect(os.environ['DATABASE_URL'],dbname=dbname,autocommit=True) as c:
|
|
c.execute(Path('/app/migrations/001_initial.sql').read_text())
|
|
c.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES ('00000000-0000-0000-0000-000000000001','benchmark','Synthetic','not-a-login-hash')")
|
|
c.execute("INSERT INTO chats(id,title) SELECT md5('chat'||n)::uuid,'chat'||n FROM generate_series(1,3) n")
|
|
c.execute("INSERT INTO members SELECT id,'00000000-0000-0000-0000-000000000001'::uuid FROM chats")
|
|
c.execute("INSERT INTO messages SELECT md5('message'||n)::uuid,md5('chat'||CASE WHEN n<=100000 THEN 1 WHEN n<=115000 THEN 2 ELSE 3 END)::uuid,'00000000-0000-0000-0000-000000000001'::uuid,repeat('payload-',20)||n,'2026-01-01'::timestamptz+n*interval '1 millisecond' FROM generate_series(1,120000) n")
|
|
c.execute('DROP INDEX messages_history')
|
|
c.execute('VACUUM ANALYZE messages')
|
|
cursor=c.execute("SELECT created_at,id FROM messages WHERE chat_id=md5('chat1')::uuid ORDER BY created_at,id OFFSET 69999 LIMIT 1").fetchone()
|
|
keyset="SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=md5('chat1')::uuid AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT 50"
|
|
offset="SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=md5('chat1')::uuid ORDER BY created_at,id OFFSET 70000 LIMIT 50"
|
|
result={'version':c.execute('SELECT version()').fetchone()[0],'rows':120000,'chat_sizes':[100000,15000,5000],'repeats':5,'plans':[],'writes':[]}
|
|
for variant,ddl in [('primary_only',None),('chat_index','CREATE INDEX experiment_idx ON messages(chat_id)'),('compound','CREATE INDEX experiment_idx ON messages(chat_id,created_at,id)')]:
|
|
if ddl: c.execute(ddl)
|
|
c.execute('ANALYZE messages')
|
|
for query_name,query,args in [('keyset',keyset,cursor),('offset',offset,())]:
|
|
for _ in range(2): c.execute(query,args).fetchall()
|
|
plans=[c.execute('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) '+query,args).fetchone()[0][0] for _ in range(5)]
|
|
result['plans'].append({'index':variant,'query':query_name,'execution_ms':[p['Execution Time'] for p in plans],'median_ms':statistics.median(p['Execution Time'] for p in plans),'plans':plans})
|
|
# Five real committed batches, then remove only those synthetic rows.
|
|
measurements=[]
|
|
for rep in range(5):
|
|
start=time.perf_counter()
|
|
c.execute("INSERT INTO messages SELECT md5('write'||n)::uuid,md5('chat1')::uuid,'00000000-0000-0000-0000-000000000001'::uuid,repeat('payload-',20)||n,'2027-01-01'::timestamptz+n*interval '1 millisecond' FROM generate_series(1,5000) n")
|
|
measurements.append(round((time.perf_counter()-start)*1000,3))
|
|
c.execute("DELETE FROM messages WHERE created_at>='2027-01-01'")
|
|
c.execute('VACUUM ANALYZE messages')
|
|
result['writes'].append({'index':variant,'batch_rows':5000,'committed_ms':measurements,'median_ms':statistics.median(measurements)})
|
|
if ddl: c.execute('DROP INDEX experiment_idx')
|
|
c.execute('CREATE INDEX messages_history ON messages(chat_id,created_at,id)')
|
|
result['snapshot']=snapshot(c)
|
|
print(json.dumps(result,default=str))
|
|
|
|
def snapshot(c):
|
|
counts={table:c.execute('SELECT count(*) FROM '+table).fetchone()[0] for table in ('users','chats','members','messages','sessions')}
|
|
selected=c.execute('SELECT id,chat_id,sender_id,text,created_at FROM messages ORDER BY id LIMIT 20').fetchall()
|
|
digest=c.execute("SELECT md5(string_agg(id::text||chat_id::text||sender_id::text||text||created_at::text,'' ORDER BY id)) FROM messages").fetchone()[0]
|
|
return {'counts':counts,'selected':[[str(v) for v in row] for row in selected],'all_messages_digest':digest}
|
|
|
|
def host():
|
|
root=Path(__file__).resolve().parents[1]
|
|
def run(*args,input=None):
|
|
return subprocess.run(['docker',*args],cwd=root,input=input,stdout=subprocess.PIPE,stderr=subprocess.PIPE,check=True).stdout
|
|
unique=uuid.uuid4().hex[:10]; dbname='experiment_'+unique
|
|
cid=run('compose','ps','-q','db').decode().strip()
|
|
source=Path(__file__).read_bytes()
|
|
result=json.loads(run('compose','exec','-T','api','python','-','--inside',dbname,input=source))
|
|
start=time.perf_counter()
|
|
dump=run('exec',cid,'pg_dump','-U','messenger','-Fc',dbname)
|
|
result['backup']={'seconds':round(time.perf_counter()-start,3),'bytes':len(dump),'sha256':hashlib.sha256(dump).hexdigest()}
|
|
backup=root/'backups'/f'{dbname}.dump'; backup.parent.mkdir(exist_ok=True); backup.write_bytes(dump)
|
|
# Distinct container and brand-new named volume: no restore onto source storage.
|
|
image=run('inspect','--format','{{.Config.Image}}',cid).decode().strip()
|
|
volume='lab2-restore-'+unique; target='lab2-restore-'+unique
|
|
run('volume','create',volume)
|
|
try:
|
|
run('run','-d','--name',target,'--cpus','1','--memory','256m','-e','POSTGRES_USER=messenger','-e','POSTGRES_DB=messenger','-e','POSTGRES_HOST_AUTH_METHOD=trust','-v',volume+':/var/lib/postgresql/data',image)
|
|
start=time.perf_counter()
|
|
for _ in range(100):
|
|
try: run('exec',target,'pg_isready','-U','messenger'); break
|
|
except subprocess.CalledProcessError: time.sleep(.2)
|
|
else: raise AssertionError('restore PostgreSQL did not become ready')
|
|
run('exec','-i',target,'pg_restore','-U','messenger','-d','messenger','--exit-on-error',input=dump)
|
|
result['restore']={'seconds_including_startup':round(time.perf_counter()-start,3),'fresh_volume':volume,'published_ports':False}
|
|
# pg_restore schema/data includes all constraints and indexes.
|
|
query="SELECT json_build_object('users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats),'members',(SELECT count(*) FROM members),'messages',(SELECT count(*) FROM messages),'sessions',(SELECT count(*) FROM sessions))"
|
|
counts=json.loads(run('exec',target,'psql','-U','messenger','-Atc',query))
|
|
digest=run('exec',target,'psql','-U','messenger','-Atc',"SELECT md5(string_agg(id::text||chat_id::text||sender_id::text||text||created_at::text,'' ORDER BY id)) FROM messages").decode().strip()
|
|
selected=json.loads(run('exec',target,'psql','-U','messenger','-Atc',"SELECT json_agg(row_to_json(t)) FROM (SELECT id,chat_id,sender_id,text,created_at::text FROM messages ORDER BY id LIMIT 20) t"))
|
|
selected=[[str(row[k]) for k in ('id','chat_id','sender_id','text','created_at')] for row in selected]
|
|
# Normalize equivalent precision/offset rendering, preserving actual instants.
|
|
for rows in (selected,result['snapshot']['selected']):
|
|
for row in rows: row[-1]=datetime.fromisoformat(row[-1]).isoformat(timespec='microseconds')
|
|
assert counts==result['snapshot']['counts']
|
|
assert digest==result['snapshot']['all_messages_digest']
|
|
assert selected==result['snapshot']['selected']
|
|
result['restore'].update(counts=counts,all_messages_digest=digest,selected_20_match=True,verified=True)
|
|
finally:
|
|
run('rm','-f',target)
|
|
run('volume','rm',volume)
|
|
result['limitations']='Warm-cache serial variants on one local host; logical backup snapshot, no WAL/PITR; synthetic DB retained for inspection.'
|
|
result['database']=dbname
|
|
(root/'evidence').mkdir(exist_ok=True)
|
|
(root/'evidence/index-restore.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
|
print(json.dumps({k:v for k,v in result.items() if k not in ('plans','snapshot')},indent=2))
|
|
|
|
if __name__=='__main__':
|
|
if '--inside' in sys.argv: inside(sys.argv[-1])
|
|
else: host()
|