forked from mtusi2026/mtusi2026lab4
307 lines
19 KiB
Python
307 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Repeatable lab4 acceptance on an isolated synthetic Compose project.
|
|
No tokens, request headers, passwords, or filled-in configuration are persisted.
|
|
"""
|
|
import concurrent.futures as futures
|
|
import datetime
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import select
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
import zlib
|
|
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0,str(ROOT/'tests'))
|
|
from client import Client, check
|
|
OUT=ROOT/'evidence'
|
|
ENV=dict(os.environ,COMPOSE_PROJECT_NAME='mtusi2026lab4-evidence',HOST_PORT='18082',UPLOAD_TIMEOUT='5s',RECONCILE_GRACE_PERIOD='15s')
|
|
COMPOSE=['./scripts/compose.sh']
|
|
BASE='http://localhost:18082'
|
|
RUN_ID=uuid.uuid4().hex
|
|
|
|
def save(name,value):
|
|
temporary=OUT/(name+'.json.tmp')
|
|
temporary.write_text(json.dumps(value,indent=2)+'\n')
|
|
temporary.replace(OUT/(name+'.json'))
|
|
|
|
def stage(name):
|
|
save('acceptance',{'passed':False,'run_id':RUN_ID,'stage':name})
|
|
print(name,flush=True)
|
|
|
|
def command(args, name=None, expected=0, **kw):
|
|
p=subprocess.run(args,cwd=ROOT,env=ENV,text=True,capture_output=True,timeout=kw.pop('timeout',240),**kw)
|
|
if name:
|
|
(OUT/(name+'.txt')).write_text('$ '+' '.join(args)+'\n'+p.stdout+p.stderr)
|
|
check(p.returncode==expected,'command failed: '+' '.join(args)+' (see evidence/'+str(name)+'.txt)')
|
|
return p.stdout
|
|
|
|
def compose(*args,**kw): return command(COMPOSE+list(args),**kw)
|
|
def operator(*args,grace=None):
|
|
options=['-e','RECONCILE_GRACE_PERIOD='+grace] if grace else []
|
|
output=compose('run','--rm','-T',*options,'operator','storage',*args)
|
|
return json.loads(output)
|
|
def sql(query): return compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',query).strip()
|
|
def snapshot():
|
|
ids=compose('ps','-q','api','postgres','redis','s3','lb').split()
|
|
results=[]
|
|
for cid in ids:
|
|
info=json.loads(command(['docker','inspect','--format','{{json .State}}',cid]))
|
|
name=command(['docker','inspect','--format','{{.Name}}',cid]).strip().lstrip('/')
|
|
results.append({'name':name,'id':cid,'started_at':info['StartedAt'],'running':info['Running'],'health':info.get('Health',{}).get('Status')})
|
|
return results
|
|
|
|
def ready(c):
|
|
deadline=time.monotonic()+90
|
|
while time.monotonic()<deadline:
|
|
try: c.call('GET','/health/ready',schema='Health'); return
|
|
except Exception: time.sleep(.5)
|
|
raise AssertionError('API did not recover')
|
|
|
|
def padded_png(data,size):
|
|
padding=b'\0'*(size-len(data)-12)
|
|
chunk=b'paDd'+padding
|
|
return data[:-12]+struct.pack('>I',len(padding))+chunk+struct.pack('>I',zlib.crc32(chunk))+data[-12:]
|
|
|
|
def upload(c,actor,chat,data):
|
|
return c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',actor=actor,raw=data,expected=(201,429),headers={'Content-Type':'image/png'},schema='Attachment')
|
|
|
|
def direct_api(container,path,actor=None,body=b'',content_type='image/png'):
|
|
# Feed credentials via stdin, never argv or evidence. Only controlled results
|
|
# are returned to the report, not the raw HTTP request.
|
|
method='POST' if body else 'GET'
|
|
headers=[f'{method} {path} HTTP/1.1','Host: api','Connection: close']
|
|
if actor: headers.append('Authorization: Bearer '+actor['token'])
|
|
if body: headers.extend(['Content-Type: '+content_type,'Content-Length: '+str(len(body))])
|
|
raw=('\r\n'.join(headers)+'\r\n\r\n').encode()+body
|
|
p=subprocess.Popen(['docker','exec','-i',container,'nc','-w','10','127.0.0.1','8080'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
|
|
try:
|
|
# BusyBox nc closes the socket on stdin EOF. Keep stdin open until the
|
|
# response starts so the probe itself does not cancel the server context.
|
|
p.stdin.write(raw);p.stdin.flush()
|
|
readable,_,_=select.select([p.stdout],[],[],15)
|
|
check(bool(readable),'internal API probe timed out')
|
|
line=p.stdout.readline()
|
|
return int(line.split(b' ',2)[1])
|
|
finally:
|
|
p.stdin.close()
|
|
try: p.wait(timeout=3)
|
|
except subprocess.TimeoutExpired: p.kill();p.wait()
|
|
|
|
def failures(c,actor,chat,data):
|
|
findings={}
|
|
att=upload(c,actor,chat,data)['json']
|
|
c.message(actor,chat,'',attachments=[att['id']])
|
|
key=sql("SELECT object_key FROM attachments WHERE id='"+att['id']+"'")
|
|
raw=compose('exec','-T','api','/usr/local/bin/messenger','storage','private-check','--key',key)
|
|
findings['private_bucket']=json.loads(raw)
|
|
save('failures',findings)
|
|
before=snapshot()
|
|
command([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh','--base-url',BASE],name='persistence')
|
|
after=snapshot()
|
|
after_by_name={x['name']:x for x in after}
|
|
check(all(old['name'] in after_by_name and after_by_name[old['name']]['running'] and
|
|
(after_by_name[old['name']]['id']!=old['id'] or after_by_name[old['name']]['started_at']!=old['started_at'])
|
|
for old in before),'restart hook did not restart or replace every store/API/LB')
|
|
findings['restart']={'before':before,'after':after}
|
|
save('failures',findings)
|
|
findings['failover']=[]
|
|
for target in compose('ps','-q','api').split():
|
|
hostname=command(['docker','inspect','--format','{{.Config.Hostname}}',target]).strip()
|
|
for _ in range(20):
|
|
response=upload(c,actor,chat,data)
|
|
if response['headers']['X-Instance-Id'].startswith(hostname+'-'):break
|
|
else: raise AssertionError('cannot route upload to intended replica')
|
|
attachment=response['json'];alias=response['headers']['X-Instance-Id']
|
|
stop_env=dict(ENV,TARGET_INSTANCE_ID=alias)
|
|
p=subprocess.run(['./scripts/stop-one.sh'],cwd=ROOT,env=stop_env,capture_output=True,timeout=15)
|
|
check(p.returncode==0,'targeted stop failed')
|
|
deadline=time.monotonic()+10
|
|
while True:
|
|
try:
|
|
downloaded=c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor)
|
|
check(downloaded['bytes']==data,'failover changed original')
|
|
check(downloaded['headers']['X-Instance-Id']!=alias,'stopped uploader served download')
|
|
break
|
|
except Exception:
|
|
check(time.monotonic()<deadline,'download did not recover');time.sleep(.3)
|
|
check(command(['docker','inspect','--format','{{.State.Running}}',target]).strip()=='false','target not stopped')
|
|
findings['failover'].append({'stopped_container':target,'upload_instance':alias,'download_instance':downloaded['headers']['X-Instance-Id'],'sha256':hashlib.sha256(data).hexdigest()})
|
|
save('failures',findings)
|
|
compose('up','-d','--no-deps','--wait','api');ready(c)
|
|
command([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh','--base-url',BASE],name='failover')
|
|
compose('up','-d','--no-deps','--wait','api');ready(c)
|
|
api=compose('ps','-q','api').split()[0]
|
|
try:
|
|
compose('stop','s3')
|
|
check(direct_api(api,'/health/live')==200,'S3 outage killed liveness')
|
|
check(direct_api(api,'/health/ready')==503,'S3 outage did not fail readiness')
|
|
status=direct_api(api,'/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',actor,data)
|
|
check(status==503,'S3 failure upload must return 503')
|
|
blocked=compose('run','--rm','-T','operator','storage','reconcile','--apply',expected=1,name='reconcile-s3-down')
|
|
scan=json.loads(blocked)
|
|
check(scan['deleted']==0 and scan['marked_missing']==0,'S3 outage caused reconciliation writes')
|
|
findings['s3_outage']={'live':200,'ready':503,'upload':status,'reconciler':scan}
|
|
save('failures',findings)
|
|
finally:
|
|
compose('up','-d','--no-deps','--wait','s3')
|
|
compose('run','--rm','-T','s3-init')
|
|
ready(c)
|
|
# A client disconnect before all bytes are received cannot create a ready row.
|
|
count_before=sql('SELECT count(*) FROM attachments')
|
|
conn=http.client.HTTPConnection('localhost',18082,timeout=10)
|
|
conn.putrequest('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png')
|
|
conn.putheader('Authorization','Bearer '+actor['token']);conn.putheader('Content-Type','image/png');conn.putheader('Content-Length',len(data)*2);conn.endheaders();conn.send(data[:100]);conn.close();time.sleep(.5)
|
|
check(sql('SELECT count(*) FROM attachments')==count_before,'interrupted body created metadata')
|
|
findings['interrupted_upload']={'metadata_unchanged':True}
|
|
save('failures',findings)
|
|
# Real database-side timeout after S3 PUT, without production fault switches.
|
|
lock=subprocess.Popen(COMPOSE+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',"BEGIN; LOCK TABLE attachments IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(4); COMMIT;"],cwd=ROOT,env=ENV,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
|
|
try:
|
|
deadline=time.monotonic()+3
|
|
while sql("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE relation='attachments'::regclass AND mode='AccessExclusiveLock' AND granted)")!='t':
|
|
check(lock.poll() is None and time.monotonic()<deadline,'lock session failed')
|
|
time.sleep(.05)
|
|
failed=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',503,actor=actor,raw=data,headers={'Content-Type':'image/png'})
|
|
check(failed['status']==503,'database timeout not reported')
|
|
finally:
|
|
lock.communicate(timeout=10)
|
|
check(sql('SELECT count(*) FROM attachments')==count_before,'failed metadata write committed')
|
|
findings['s3_put_db_timeout']={'upload_status':503,'metadata_unchanged':True}
|
|
save('failures',findings)
|
|
findings['dry_run_after_failed_commit']=operator('reconcile')
|
|
check(findings['dry_run_after_failed_commit']['orphans']>=1,'failed DB write did not leave detectable orphan')
|
|
# Missing objects retain IDs/references and can only be restored from originals.
|
|
operator('remove-object','--id',att['id'])
|
|
dry=operator('reconcile');check(dry['missing']>=1,'missing object not detected')
|
|
apply=operator('reconcile','--apply');check(apply['missing']>=1,'missing object not marked')
|
|
c.call('GET','/api/v1/attachments/'+att['id'],503,actor=actor)
|
|
bad=compose('run','--rm','-T','operator','storage','restore','--id',att['id'],'--source','/fixtures/README.md',expected=1)
|
|
operator('restore','--id',att['id'],'--source','/fixtures/sample.png')
|
|
operator('restore','--id',att['id'],'--source','/fixtures/sample.png')
|
|
downloaded=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)
|
|
check(downloaded['bytes']==data,'restored bytes changed')
|
|
history=c.call('GET','/api/v1/chats/'+chat['id']+'/messages',actor=actor,schema='MessagePage')['json']
|
|
check(any(att['id'] in m['attachment_ids'] for m in history['items']),'restoration lost message reference')
|
|
findings['missing_repair']={'dry_run':dry,'apply':apply,'wrong_original_rejected':True,'restored_sha256':hashlib.sha256(data).hexdigest(),'reference_preserved':True,'repeat_safe':True}
|
|
save('failures',findings)
|
|
count_before=sql('SELECT count(*) FROM attachments')
|
|
try:
|
|
compose('stop','postgres')
|
|
raw=compose('run','--rm','-T','operator','storage','reconcile','--apply',expected=1,name='reconcile-db-down')
|
|
scan=json.loads(raw)
|
|
check(scan['deleted']==0 and scan['marked_missing']==0,'DB outage caused reconciliation writes')
|
|
findings['database_outage']={'reconciler':scan}
|
|
finally:
|
|
compose('up','-d','--no-deps','--wait','postgres');ready(c)
|
|
check(sql('SELECT count(*) FROM attachments')==count_before,'DB restart lost attachment metadata')
|
|
save('failures',findings)
|
|
return findings
|
|
|
|
def measurements(c,actor,chat,data):
|
|
records=[]
|
|
for size in [len(data),5*1024*1024]:
|
|
body=data if size==len(data) else padded_png(data,size)
|
|
for trial in range(1,4):
|
|
compose('up','-d','--no-deps','--force-recreate','--wait','api');ready(c)
|
|
upload(c,actor,chat,data) # One documented warm-up upload.
|
|
ids=compose('ps','-q','api').split()
|
|
cpu=subprocess.Popen(['docker','stats','--no-stream','--format','{{json .}}',*ids],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
|
|
barrier=threading.Barrier(10)
|
|
def work(_): barrier.wait();return upload(Client(BASE,timeout=30),actor,chat,body)
|
|
start=time.monotonic()
|
|
with futures.ThreadPoolExecutor(max_workers=10) as pool: results=list(pool.map(work,range(10)))
|
|
duration=time.monotonic()-start
|
|
successful=[x['elapsed'] for x in results if x['status']==201]
|
|
rss=[]
|
|
for cid in ids:
|
|
values=command(['docker','exec',cid,'sh','-c','cat /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.peak']).split()
|
|
rss.append({'container':cid,'memory_current_bytes':int(values[0]),'memory_peak_bytes':int(values[1])})
|
|
stats,_=cpu.communicate(timeout=15)
|
|
cpu_samples=[{'container':v.get('Name'),'cpu_percent':v.get('CPUPerc')} for v in [json.loads(line) for line in stats.splitlines()]]
|
|
def quantile(p): return sorted(successful)[min(len(successful)-1,int((len(successful)-1)*p))] if successful else None
|
|
records.append({'bytes':len(body),'trial':trial,'clients':10,'warmup_uploads':1,'duration_seconds':duration,'successful':len(successful),'errors':10-len(successful),'error_rate':(10-len(successful))/10,'successful_rps':len(successful)/duration,'p50':quantile(.5),'p95':quantile(.95),'p99':quantile(.99),'status_counts':{str(code):sum(x['status']==code for x in results) for code in [201,429]},'memory':rss,'cpu':cpu_samples})
|
|
save('uploads',records)
|
|
return records
|
|
|
|
def research(c,actor,chat,data):
|
|
results=[]
|
|
# Demonstrate freshness independently of the duration of mass seeding/scanning.
|
|
fresh_key=operator('seed')['keys'][0]
|
|
invalid_key='originals/not-a-server-uuid-'+uuid.uuid4().hex
|
|
outside_key='unmanaged/'+uuid.uuid4().hex
|
|
operator('seed','--key',invalid_key)
|
|
operator('seed','--key',outside_key)
|
|
fresh=operator('reconcile','--apply',grace='1h')
|
|
check(operator('object-check','--key',fresh_key)['exists'],'fresh object deleted')
|
|
# Upload and reconciliation execute concurrently; every acknowledged upload
|
|
# must remain downloadable. The separate PostgreSQL test checks lock contention.
|
|
with futures.ThreadPoolExecutor(max_workers=2) as workers:
|
|
scan=workers.submit(operator,'reconcile','--apply')
|
|
concurrent_upload=upload(c,actor,chat,data)
|
|
scan_result=scan.result()
|
|
check(concurrent_upload['status']==201,'concurrent upload failed')
|
|
att=concurrent_upload['json']
|
|
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)['bytes']==data,'concurrent reconciliation damaged upload')
|
|
safety={'fresh_apply':fresh,'fresh_key_preserved':True,'concurrent_scan':scan_result,'concurrent_upload_sha256':hashlib.sha256(data).hexdigest()}
|
|
save('reconciliation-safety',safety)
|
|
for count in [100,1000]:
|
|
operator('seed','--count',str(count))
|
|
# All objects exist before the grace wait starts. Duration of seeding is
|
|
# intentionally excluded from the scan timing, but dataset size is recorded.
|
|
time.sleep(16)
|
|
scans=[operator('reconcile') for _ in range(3)]
|
|
applied=operator('reconcile','--apply')
|
|
repeated=operator('reconcile','--apply')
|
|
check(applied['deleted']>=count,'old orphans not cleaned')
|
|
check(repeated['deleted']==0 and repeated['missing']==0,'repeat was not safe or valid original deleted')
|
|
check(operator('object-check','--key',invalid_key)['exists'],'invalid managed key deleted')
|
|
check(operator('object-check','--key',outside_key)['exists'],'object outside prefix deleted')
|
|
results.append({'seeded':count,'dry_runs':scans,'apply':applied,'repeat_apply':repeated,'invalid_key_preserved':True,'outside_prefix_preserved':True})
|
|
save('reconciliation',results)
|
|
return results
|
|
|
|
def main():
|
|
OUT.mkdir(exist_ok=True)
|
|
stage("bootstrap")
|
|
command(['./scripts/bootstrap.sh'],name='bootstrap')
|
|
before_keys=hashlib.sha256((ROOT/'secrets/lab4/s3.json').read_bytes()).hexdigest()
|
|
command(['./scripts/bootstrap.sh'],name='bootstrap-repeat')
|
|
check(before_keys==hashlib.sha256((ROOT/'secrets/lab4/s3.json').read_bytes()).hexdigest(),'bootstrap rotated credentials')
|
|
compose('config','--quiet',name='compose-config')
|
|
compose('up','-d','--build','--wait','--wait-timeout','120',name='stand-up',timeout=600)
|
|
meta={'run_id':RUN_ID,'date_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'git_head':command(['git','rev-parse','HEAD']).strip(),'working_tree_dirty':bool(command(['git','status','--porcelain']).strip()),'host':platform.platform(),'docker':json.loads(command(['docker','info','--format','{{json .}}'])),'project':ENV['COMPOSE_PROJECT_NAME'],'limits':{'api_replicas':2,'api_cpu_each':1,'api_ram_each':536870912,'upload_timeout':'5s','grace':'15s','upload_slots_each':4,'decode_slots_each':1}}
|
|
# Keep a small allow-list; docker info can contain unrelated host settings.
|
|
meta['docker']={k:meta['docker'][k] for k in ['ServerVersion','Architecture','NCPU','MemTotal']}
|
|
(OUT/'environment.json').write_text(json.dumps(meta,indent=2)+'\n')
|
|
compose('run','--rm','-T','s3-init',name='bucket-bootstrap-repeat')
|
|
c=Client(BASE,timeout=20);ready(c)
|
|
command(['make','check','build','vet'],name='build-checks')
|
|
validator=str(ROOT/'.venv/bin/python') if (ROOT/'.venv/bin/python').exists() else sys.executable
|
|
command(['make','validate','PYTHON='+validator],name='validate')
|
|
command(['make','test-go','BASE_URL='+BASE],name='go-tests')
|
|
command(['make','test-storage'],name='storage-tests')
|
|
command(['make','test','BASE_URL='+BASE],name='smoke')
|
|
actor,bob,eve,chat=c.bootstrap();data=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
stage('faults')
|
|
findings=failures(c,actor,chat,data)
|
|
(OUT/'failures.json').write_text(json.dumps(findings,indent=2)+'\n')
|
|
stage('uploads')
|
|
(OUT/'uploads.json').write_text(json.dumps(measurements(c,actor,chat,data),indent=2)+'\n')
|
|
stage('reconciliation')
|
|
(OUT/'reconciliation.json').write_text(json.dumps(research(c,actor,chat,data),indent=2)+'\n')
|
|
result={'run_id':RUN_ID,'passed':True,'checks':['contract','go_race','transaction_rollback','smoke','private_bucket','persistence','both_uploaders_failover','public_failover','s3_down','interrupted_body','s3_put_db_timeout','missing_restore','10_client_memory','reconciler_grace','reconciler_repeat'],'remaining_metadata':int(sql('SELECT count(*) FROM attachments')),'timestamp_utc':datetime.datetime.now(datetime.timezone.utc).isoformat()}
|
|
compose('down',name='stand-down')
|
|
save('acceptance',result) # Preserve all experiment volumes.
|
|
print('PASS: lab4 acceptance; summaries in evidence/. Isolated stand stopped; volumes preserved.',flush=True)
|
|
|
|
if __name__=='__main__': main()
|