forked from mtusi2026/mtusi2026lab2
104 lines
6.0 KiB
Python
104 lines
6.0 KiB
Python
"""Measure indexes, back up, restore to a NEW PostgreSQL container+volume, verify."""
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
IMAGE='postgres:17.9-bookworm@sha256:47f917f7409eacd22fc5dfb1dee634e1b55cf0c01d1a7eb701be2227a03e0641'
|
|
|
|
|
|
def execute(command,**kwargs):
|
|
return subprocess.run(command,cwd=ROOT,check=True,capture_output=True,**kwargs)
|
|
|
|
|
|
def compose(*args,**kwargs):
|
|
return execute(['docker','compose',*args],**kwargs)
|
|
|
|
|
|
def main():
|
|
prefix='lab2_research_'+uuid.uuid4().hex[:8]
|
|
target='mtusi-lab2-restore-'+uuid.uuid4().hex[:8]
|
|
volume=target+'-data'
|
|
worker=(ROOT/'research/worker.py').read_text()
|
|
def action(name,**variables):
|
|
flags=[]
|
|
for key,value in {'RESEARCH_ACTION':name,'RESEARCH_PREFIX':prefix,**variables}.items():
|
|
flags.extend(['-e',key+'='+value])
|
|
return json.loads(compose('exec','-T',*flags,'api','python','-',input=worker,text=True).stdout)
|
|
report={'date_utc':dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
'source_sha256':{str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in [ROOT/'src/store.py',ROOT/'migrations/001_initial.sql',ROOT/'research/worker.py']},
|
|
'method':'Isolated skewed PostgreSQL datasets; 30 sequential reads after 3 warmups; 10 committed INSERT batches of 1000 rows; fresh restore volume.',
|
|
'limitations':['Local single-host measurements; warm buffers; no concurrent HTTP capacity claim.',
|
|
'EXPLAIN insert wall times include plan collection and COMMIT; 10 batches do not establish a write SLA.',
|
|
'Logical backup is a snapshot, not PITR or HA; the post-dump canary is intentionally absent after restore.']}
|
|
print('Benchmarking 100000 messages, 4 skewed chats, 3 indexes and 2 queries...',flush=True)
|
|
try:
|
|
report['query_experiment']=action('benchmark')
|
|
source=report['query_experiment']['backup_source']
|
|
backup_dir=ROOT/'backups'
|
|
backup_dir.mkdir(mode=0o700,exist_ok=True)
|
|
dump=backup_dir/'lab2-research.dump'
|
|
started=time.perf_counter()
|
|
with dump.open('wb') as output:
|
|
subprocess.run(['docker','compose','exec','-T','postgres','pg_dump','-U','messenger','-d',source,'-Fc'],cwd=ROOT,stdout=output,stderr=subprocess.PIPE,check=True)
|
|
dump.chmod(0o600)
|
|
backup_seconds=time.perf_counter()-started
|
|
report['after_snapshot_write']=action('canary',RESEARCH_SOURCE=source)
|
|
cid=compose('ps','-q','postgres',text=True).stdout.strip()
|
|
inspect=json.loads(execute(['docker','inspect',cid],text=True).stdout)[0]
|
|
networks=list(inspect['NetworkSettings']['Networks'])
|
|
assert len(networks)==1
|
|
provision_start=time.perf_counter()
|
|
execute(['docker','volume','create',volume],text=True)
|
|
execute(['docker','run','-d','--name',target,'--network',networks[0],
|
|
'--cpus=1','--memory=512m','--memory-swap=512m','--pids-limit=128',
|
|
'--mount','type=volume,source='+volume+',target=/var/lib/postgresql/data',
|
|
'--mount','type=bind,source='+str(ROOT/'secrets/db_password')+',target=/run/secrets/db_password,readonly',
|
|
'-e','POSTGRES_DB=messenger','-e','POSTGRES_USER=messenger','-e','POSTGRES_PASSWORD_FILE=/run/secrets/db_password',IMAGE],text=True)
|
|
deadline=time.monotonic()+30
|
|
while True:
|
|
status=subprocess.run(['docker','exec',target,'pg_isready','-h','127.0.0.1','-U','messenger','-d','messenger'],capture_output=True)
|
|
if status.returncode==0:
|
|
break
|
|
if time.monotonic()>deadline:
|
|
raise RuntimeError('new restore PostgreSQL did not become ready')
|
|
time.sleep(.2)
|
|
provisioning=time.perf_counter()-provision_start
|
|
started=time.perf_counter()
|
|
with dump.open('rb') as input_file:
|
|
subprocess.run(['docker','exec','-i',target,'pg_restore','--exit-on-error','--no-owner','--no-privileges','-U','messenger','-d','messenger'],stdin=input_file,stdout=subprocess.PIPE,stderr=subprocess.PIPE,check=True)
|
|
restore_seconds=time.perf_counter()-started
|
|
verified=action('verify',RESTORE_HOST=target)
|
|
expected=report['query_experiment']['before_backup']
|
|
assert verified['counts']==expected['counts']
|
|
assert verified['all_messages_sha256']==expected['all_messages_sha256']
|
|
assert verified['selected_messages']==expected['selected_messages']
|
|
assert verified['orphan_messages']==0
|
|
assert verified['counts']['messages']==100000
|
|
report['backup_restore']={'backup_seconds':round(backup_seconds,4),'dump_bytes':dump.stat().st_size,
|
|
'dump_sha256':hashlib.sha256(dump.read_bytes()).hexdigest(),
|
|
'new_container_and_volume':True,'provision_seconds':round(provisioning,4),
|
|
'restore_seconds':round(restore_seconds,4),'provision_plus_restore_seconds':round(provisioning+restore_seconds,4),
|
|
'verification':verified,'all_counts_and_samples_equal':True,'all_messages_checksum_equal':True,
|
|
'after_backup_canary_intentionally_absent':True}
|
|
report['status']='PASS'
|
|
destination=ROOT/'evidence/research.json'
|
|
destination.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
|
print('PASS: restored 100000 messages to a new PostgreSQL container/volume; counts, 7 samples, complete SHA256 and foreign keys verified.',flush=True)
|
|
execute([sys.executable,'research/summarize.py'],text=True)
|
|
finally:
|
|
subprocess.run(['docker','rm','-f',target],capture_output=True)
|
|
# This volume was created by this experiment and contains synthetic data only.
|
|
subprocess.run(['docker','volume','rm',volume],capture_output=True)
|
|
action('cleanup')
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|