Files
mtusi2026lab2/research/run.py
T

104 lines
6.0 KiB
Python

"""Measure indexes, back up, restore to a NEW PostgreSQL container+volume, verify."""
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
import uuid
ROOT=Path(__file__).resolve().parents[1]
IMAGE='postgres:17.9-bookworm@sha256:47f917f7409eacd22fc5dfb1dee634e1b55cf0c01d1a7eb701be2227a03e0641'
def execute(command,**kwargs):
return subprocess.run(command,cwd=ROOT,check=True,capture_output=True,**kwargs)
def compose(*args,**kwargs):
return execute(['docker','compose',*args],**kwargs)
def main():
prefix='lab2_research_'+uuid.uuid4().hex[:8]
target='mtusi-lab2-restore-'+uuid.uuid4().hex[:8]
volume=target+'-data'
worker=(ROOT/'research/worker.py').read_text()
def action(name,**variables):
flags=[]
for key,value in {'RESEARCH_ACTION':name,'RESEARCH_PREFIX':prefix,**variables}.items():
flags.extend(['-e',key+'='+value])
return json.loads(compose('exec','-T',*flags,'api','python','-',input=worker,text=True).stdout)
report={'date_utc':dt.datetime.now(dt.timezone.utc).isoformat(),
'source_sha256':{str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in [ROOT/'src/store.py',ROOT/'migrations/001_initial.sql',ROOT/'research/worker.py']},
'method':'Isolated skewed PostgreSQL datasets; 30 sequential reads after 3 warmups; 10 committed INSERT batches of 1000 rows; fresh restore volume.',
'limitations':['Local single-host measurements; warm buffers; no concurrent HTTP capacity claim.',
'EXPLAIN insert wall times include plan collection and COMMIT; 10 batches do not establish a write SLA.',
'Logical backup is a snapshot, not PITR or HA; the post-dump canary is intentionally absent after restore.']}
print('Benchmarking 100000 messages, 4 skewed chats, 3 indexes and 2 queries...',flush=True)
try:
report['query_experiment']=action('benchmark')
source=report['query_experiment']['backup_source']
backup_dir=ROOT/'backups'
backup_dir.mkdir(mode=0o700,exist_ok=True)
dump=backup_dir/'lab2-research.dump'
started=time.perf_counter()
with dump.open('wb') as output:
subprocess.run(['docker','compose','exec','-T','postgres','pg_dump','-U','messenger','-d',source,'-Fc'],cwd=ROOT,stdout=output,stderr=subprocess.PIPE,check=True)
dump.chmod(0o600)
backup_seconds=time.perf_counter()-started
report['after_snapshot_write']=action('canary',RESEARCH_SOURCE=source)
cid=compose('ps','-q','postgres',text=True).stdout.strip()
inspect=json.loads(execute(['docker','inspect',cid],text=True).stdout)[0]
networks=list(inspect['NetworkSettings']['Networks'])
assert len(networks)==1
provision_start=time.perf_counter()
execute(['docker','volume','create',volume],text=True)
execute(['docker','run','-d','--name',target,'--network',networks[0],
'--cpus=1','--memory=512m','--memory-swap=512m','--pids-limit=128',
'--mount','type=volume,source='+volume+',target=/var/lib/postgresql/data',
'--mount','type=bind,source='+str(ROOT/'secrets/db_password')+',target=/run/secrets/db_password,readonly',
'-e','POSTGRES_DB=messenger','-e','POSTGRES_USER=messenger','-e','POSTGRES_PASSWORD_FILE=/run/secrets/db_password',IMAGE],text=True)
deadline=time.monotonic()+30
while True:
status=subprocess.run(['docker','exec',target,'pg_isready','-h','127.0.0.1','-U','messenger','-d','messenger'],capture_output=True)
if status.returncode==0:
break
if time.monotonic()>deadline:
raise RuntimeError('new restore PostgreSQL did not become ready')
time.sleep(.2)
provisioning=time.perf_counter()-provision_start
started=time.perf_counter()
with dump.open('rb') as input_file:
subprocess.run(['docker','exec','-i',target,'pg_restore','--exit-on-error','--no-owner','--no-privileges','-U','messenger','-d','messenger'],stdin=input_file,stdout=subprocess.PIPE,stderr=subprocess.PIPE,check=True)
restore_seconds=time.perf_counter()-started
verified=action('verify',RESTORE_HOST=target)
expected=report['query_experiment']['before_backup']
assert verified['counts']==expected['counts']
assert verified['all_messages_sha256']==expected['all_messages_sha256']
assert verified['selected_messages']==expected['selected_messages']
assert verified['orphan_messages']==0
assert verified['counts']['messages']==100000
report['backup_restore']={'backup_seconds':round(backup_seconds,4),'dump_bytes':dump.stat().st_size,
'dump_sha256':hashlib.sha256(dump.read_bytes()).hexdigest(),
'new_container_and_volume':True,'provision_seconds':round(provisioning,4),
'restore_seconds':round(restore_seconds,4),'provision_plus_restore_seconds':round(provisioning+restore_seconds,4),
'verification':verified,'all_counts_and_samples_equal':True,'all_messages_checksum_equal':True,
'after_backup_canary_intentionally_absent':True}
report['status']='PASS'
destination=ROOT/'evidence/research.json'
destination.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
print('PASS: restored 100000 messages to a new PostgreSQL container/volume; counts, 7 samples, complete SHA256 and foreign keys verified.',flush=True)
execute([sys.executable,'research/summarize.py'],text=True)
finally:
subprocess.run(['docker','rm','-f',target],capture_output=True)
# This volume was created by this experiment and contains synthetic data only.
subprocess.run(['docker','volume','rm',volume],capture_output=True)
action('cleanup')
if __name__=='__main__':
main()