228 lines
13 KiB
Python
228 lines
13 KiB
Python
"""Grade 5, track 2: isolated index experiment and verified PostgreSQL restore.
|
|
|
|
Only synthetic data, standard library + Docker. Main Compose volumes are untouched.
|
|
"""
|
|
import argparse
|
|
import csv
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import statistics
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
|
|
from infrastructure import ROOT, docker
|
|
|
|
IMAGE = 'postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3'
|
|
|
|
|
|
def key(kind, number):
|
|
# Deterministic fixture UUIDs only; password hashing remains Argon2id in auth.go.
|
|
return str(uuid.UUID(hashlib.md5(f'lab2-{kind}-{number}'.encode()).hexdigest()))
|
|
|
|
|
|
def sql(container, query):
|
|
return docker('exec', '-i', container, 'psql', '-XqAt', '-v', 'ON_ERROR_STOP=1',
|
|
'-U', 'messenger', '-d', 'messenger', data=query)
|
|
|
|
|
|
def plans(container, query):
|
|
raw = sql(container, query).strip()
|
|
result, decoder = [], json.JSONDecoder()
|
|
while raw:
|
|
value, end = decoder.raw_decode(raw)
|
|
result.append(value[0])
|
|
raw = raw[end:].lstrip()
|
|
return result
|
|
|
|
|
|
def start(name):
|
|
docker('run', '-d', '--name', name, '--network', 'none', '--memory', '512m', '--cpus', '2',
|
|
'-e', 'POSTGRES_HOST_AUTH_METHOD=trust', '-e', 'POSTGRES_USER=messenger',
|
|
'-e', 'POSTGRES_DB=messenger', '--mount', f'type=volume,source={name}-data,target=/var/lib/postgresql/data',
|
|
IMAGE, '-c', 'max_parallel_workers_per_gather=0', '-c', 'jit=off')
|
|
deadline = time.monotonic() + 45
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
# initdb temporarily starts a socket-only server: require final PID 1.
|
|
docker('exec', name, 'pg_isready', '-h', '127.0.0.1', '-U', 'messenger')
|
|
return
|
|
except AssertionError:
|
|
time.sleep(.2)
|
|
raise RuntimeError('PostgreSQL startup deadline exceeded')
|
|
|
|
|
|
def fingerprint(container):
|
|
query = """COPY (SELECT id,chat_id,sender_id,encode(text,'hex'),
|
|
to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS.US')
|
|
FROM messages ORDER BY id) TO STDOUT;"""
|
|
command = ['docker', 'exec', container, 'psql', '-XqAt', '-v', 'ON_ERROR_STOP=1',
|
|
'-U', 'messenger', '-d', 'messenger', '-c', query]
|
|
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
digest = hashlib.sha256()
|
|
for chunk in iter(lambda: process.stdout.read(1024*1024), b''):
|
|
digest.update(chunk)
|
|
error = process.stderr.read().decode('utf-8', errors='replace')
|
|
if process.wait() != 0:
|
|
raise RuntimeError('Fingerprint failed: '+error[-300:])
|
|
return digest.hexdigest()
|
|
|
|
|
|
def snapshot(container):
|
|
counts = {table: int(sql(container, f'SELECT count(*) FROM {table};'))
|
|
for table in ('users', 'chats', 'chat_members', 'messages', 'schema_migrations')}
|
|
ids = ','.join("'"+key('message', n)+"'" for n in (1, 2, 40000, 79999, 80000, 80001, 90000, 95000, 95001, 99999, 100000))
|
|
sample = json.loads(sql(container, f"""SELECT json_agg(t ORDER BY id) FROM (
|
|
SELECT id,chat_id,sender_id,encode(text,'hex') AS utf8_hex,created_at FROM messages
|
|
WHERE id IN ({ids})) t;"""))
|
|
assert len(sample) == 11
|
|
chat_counts = json.loads(sql(container,"SELECT json_object_agg(chat_id,n) FROM (SELECT chat_id,count(*) n FROM messages GROUP BY chat_id) t;"))
|
|
assert sorted(chat_counts.values()) == [5000,15000,80000]
|
|
return {'counts': counts, 'chat_counts':chat_counts, 'history_sha256': fingerprint(container),
|
|
'samples': {row['id']: hashlib.sha256(json.dumps(row,sort_keys=True).encode()).hexdigest() for row in sample}}
|
|
|
|
|
|
def percentile(values, fraction):
|
|
return sorted(values)[math.ceil(len(values)*fraction)-1]
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--keep', action='store_true', help='Keep the two temporary containers/volumes for inspection')
|
|
args = parser.parse_args()
|
|
suffix = uuid.uuid4().hex[:10]
|
|
source, target = 'lab2-research-'+suffix, 'lab2-restore-'+suffix
|
|
created, raw, saved_plans = [], [], []
|
|
user, chats = key('user', 1), [key('chat', n) for n in (1, 2, 3)]
|
|
evidence = ROOT/'evidence'
|
|
evidence.mkdir(exist_ok=True)
|
|
out = {'utc':dt.datetime.now(dt.timezone.utc).isoformat(), 'image':IMAGE,
|
|
'host':docker('info','--format','{{.OSType}}/{{.Architecture}} CPUs={{.NCPU}} MemoryBytes={{.MemTotal}}'),
|
|
'limits':{'cpus_per_container':2,'memory_bytes':512*1024*1024,'clients':1},
|
|
'method':{'repeats':3,'read_samples_per_case_per_repeat':20,'warmup':3,
|
|
'order':['chat,composite','composite,chat','chat,composite'],
|
|
'distribution':[80000,15000,5000],'parallel_query':False,'jit':False}}
|
|
insert = f"""INSERT INTO messages(id,chat_id,sender_id,text,created_at)
|
|
SELECT md5('lab2-message-'||g)::uuid,
|
|
CASE WHEN g<=80000 THEN '{chats[0]}'::uuid WHEN g<=95000 THEN '{chats[1]}'::uuid ELSE '{chats[2]}'::uuid END,
|
|
'{user}'::uuid,convert_to('Синтетическое сообщение №'||g||repeat(' x',40),'UTF8')
|
|
||CASE WHEN g%10000=0 THEN decode('00','hex') ELSE ''::bytea END,
|
|
'2026-01-01T00:00:00Z'::timestamptz+g*interval '1 millisecond'
|
|
FROM generate_series(1,100000) g;"""
|
|
try:
|
|
created.append(source)
|
|
start(source)
|
|
out['postgres'] = sql(source,'SELECT version();')
|
|
sql(source,'CREATE TABLE schema_migrations(version text PRIMARY KEY,checksum text NOT NULL,applied_at timestamptz NOT NULL DEFAULT clock_timestamp());')
|
|
for migration in sorted((ROOT/'migrations').glob('*.sql')):
|
|
content = migration.read_bytes()
|
|
checksum = hashlib.sha256(content).hexdigest()
|
|
sql(source, "BEGIN;\n"+content.decode('utf-8')+f"\nINSERT INTO schema_migrations(version,checksum) VALUES('{migration.name}','{checksum}'); COMMIT;")
|
|
# Valid PHC shape, no known password: this fixture user is never used for login.
|
|
fixture_hash = '$argon2id$v=19$m=65536,t=3,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
|
|
sql(source,f"INSERT INTO users(id,username,display_name,password_hash) VALUES('{user}','research_user',convert_to('Synthetic fixture','UTF8'),'{fixture_hash}');")
|
|
for index, chat in enumerate(chats):
|
|
sql(source,f"INSERT INTO chats(id,title) VALUES('{chat}',convert_to('research {index}','UTF8')); INSERT INTO chat_members VALUES('{chat}','{user}');")
|
|
cases = [('hot_first',chats[0],0),('hot_deep',chats[0],70000),('small_deep',chats[2],99000)]
|
|
for repeat in range(3):
|
|
for variant in (('chat','composite') if repeat%2==0 else ('composite','chat')):
|
|
index_name = 'research_chat_idx' if variant=='chat' else 'messages_history_idx'
|
|
columns = 'chat_id' if variant=='chat' else 'chat_id,created_at,id'
|
|
sql(source,f'TRUNCATE messages; DROP INDEX IF EXISTS messages_history_idx; DROP INDEX IF EXISTS research_chat_idx; CREATE INDEX {index_name} ON messages({columns});')
|
|
before = time.monotonic()
|
|
write = plans(source,'EXPLAIN (ANALYZE, BUFFERS, WAL, FORMAT JSON) '+insert)[0]
|
|
wall = (time.monotonic()-before)*1000
|
|
assert sql(source,'SELECT count(*) FROM messages;')=='100000'
|
|
sql(source,'ANALYZE messages;')
|
|
size = int(sql(source,f"SELECT pg_relation_size('{index_name}');"))
|
|
raw.append({'repeat':repeat+1,'variant':variant,'case':'insert_100000',
|
|
'execution_ms':write['Execution Time'],'wall_with_commit_and_docker_ms':wall,
|
|
'wal_bytes':write['Plan'].get('WAL Bytes',0),'index_bytes':size})
|
|
if repeat==0:
|
|
saved_plans.append(variant+'/insert_100000 (EXPLAIN JSON)\n'+json.dumps(write,ensure_ascii=False))
|
|
for case, chat, number in cases:
|
|
boundary = key('message',number) if number else '00000000-0000-0000-0000-000000000000'
|
|
query = f"""EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)
|
|
SELECT id::text,chat_id::text,sender_id::text,text,created_at FROM messages
|
|
WHERE chat_id='{chat}' AND (created_at,id)>(
|
|
'2026-01-01T00:00:00Z'::timestamptz+{number}*interval '1 millisecond','{boundary}'::uuid)
|
|
ORDER BY messages.created_at,messages.id LIMIT 51;"""
|
|
plans(source,query*3)
|
|
measured = plans(source,query*20)
|
|
for plan in measured:
|
|
assert plan['Plan']['Actual Rows']==51
|
|
raw.append({'repeat':repeat+1,'variant':variant,'case':case,
|
|
'execution_ms':plan['Execution Time'],'planning_ms':plan['Planning Time'],
|
|
'shared_hit_blocks':plan['Plan'].get('Shared Hit Blocks',0),
|
|
'shared_read_blocks':plan['Plan'].get('Shared Read Blocks',0)})
|
|
if repeat==0:
|
|
saved_plans.append(variant+'/'+case+'\n'+query+'\n'+sql(source,query.replace(', FORMAT JSON','')))
|
|
print(f'RESEARCH: repeat {repeat+1}/3, {variant}, 100000 rows, insert {write["Execution Time"]:.1f} ms',flush=True)
|
|
summary=[]
|
|
for variant in ('chat','composite'):
|
|
for case in ('insert_100000','hot_first','hot_deep','small_deep'):
|
|
samples=[r for r in raw if r['variant']==variant and r['case']==case]
|
|
values=[r['execution_ms'] for r in samples]
|
|
row={'variant':variant,'case':case,'samples':len(values),'median_ms':statistics.median(values),
|
|
'p95_ms':percentile(values,.95) if len(values)>3 else None,
|
|
'p99_ms':percentile(values,.99) if len(values)>3 else None}
|
|
for metric in ('index_bytes','wal_bytes','shared_hit_blocks','shared_read_blocks'):
|
|
row[metric]=statistics.median([s[metric] for s in samples]) if metric in samples[0] else None
|
|
summary.append(row)
|
|
out['summary']=summary
|
|
expected = snapshot(source)
|
|
assert expected['counts']['messages']==100000
|
|
backup_dir = ROOT/'backups'
|
|
backup_dir.mkdir(exist_ok=True)
|
|
archive = backup_dir/f'research-{suffix}.dump'
|
|
before = time.monotonic()
|
|
docker('exec',source,'pg_dump','-U','messenger','-d','messenger','-Fc','-f','/tmp/research.dump')
|
|
dump_seconds = time.monotonic()-before
|
|
docker('cp',source+':/tmp/research.dump',str(archive))
|
|
marker = key('post-backup',1)
|
|
sql(source,f"INSERT INTO messages VALUES('{marker}','{chats[0]}','{user}',convert_to('Committed after backup','UTF8'),clock_timestamp());")
|
|
assert sql(source,'SELECT count(*) FROM messages;')=='100001'
|
|
recovery_start = time.monotonic()
|
|
created.append(target)
|
|
start(target)
|
|
startup_seconds = time.monotonic()-recovery_start
|
|
assert sql(target,"SELECT count(*) FROM pg_tables WHERE schemaname='public';")=='0'
|
|
docker('cp',str(archive),target+':/tmp/research.dump')
|
|
before = time.monotonic()
|
|
docker('exec',target,'pg_restore','--exit-on-error','-U','messenger','-d','messenger','/tmp/research.dump')
|
|
restore_seconds = time.monotonic()-before
|
|
actual = snapshot(target)
|
|
assert actual==expected, 'Restored counts, selected messages or complete history differ'
|
|
assert sql(target,f"SELECT count(*) FROM messages WHERE id='{marker}';")=='0'
|
|
out['restore']={'source_container':source,'target_container':target,
|
|
'source_volume':source+'-data','new_volume':target+'-data',
|
|
'destination_initial_public_tables':0,'dump_seconds':dump_seconds,
|
|
'dump_bytes':archive.stat().st_size,'dump_path':archive.relative_to(ROOT).as_posix(),
|
|
'destination_startup_seconds':startup_seconds,'pg_restore_seconds':restore_seconds,
|
|
'recovery_through_verification_seconds':time.monotonic()-recovery_start,
|
|
'before_backup':expected,'after_restore':actual,
|
|
'source_count_after_backup':100001,'restored_count':100000,'post_backup_marker_absent':True}
|
|
out['result']='PASS: two indexes, 100000 uneven messages, write cost, new-volume restore and SHA256 verified'
|
|
(evidence/'research.json').write_text(json.dumps(out,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
|
(evidence/'research-plans.txt').write_text('\n\n'.join(saved_plans)+'\n',encoding='utf-8')
|
|
with (evidence/'research.csv').open('w',encoding='utf-8',newline='') as file:
|
|
writer=csv.DictWriter(file,fieldnames=list(dict.fromkeys(k for row in raw for k in row)))
|
|
writer.writeheader()
|
|
writer.writerows(raw)
|
|
print(out['result'],flush=True)
|
|
print(f'RESTORE: {restore_seconds:.3f}s; same 100000 rows, 11 samples and full SHA256; later COMMIT absent',flush=True)
|
|
if args.keep:
|
|
print('KEPT: '+source+' '+target,flush=True)
|
|
finally:
|
|
if not args.keep:
|
|
for name in reversed(created):
|
|
subprocess.run(['docker','rm','-f',name],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
subprocess.run(['docker','volume','rm',name+'-data'],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|