Files
mtusi2026lab4/scripts/acceptance.py
2026-11-13 11:05:05 +03:00

307 lines
19 KiB
Python

#!/usr/bin/env python3
"""Repeatable lab4 acceptance on an isolated synthetic Compose project.
No tokens, request headers, passwords, or filled-in configuration are persisted.
"""
import concurrent.futures as futures
import datetime
import hashlib
import http.client
import json
import os
from pathlib import Path
import platform
import select
import struct
import subprocess
import sys
import threading
import time
import uuid
import zlib
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client, check
OUT=ROOT/'evidence'
ENV=dict(os.environ,COMPOSE_PROJECT_NAME='mtusi2026lab4-evidence',HOST_PORT='18082',UPLOAD_TIMEOUT='5s',RECONCILE_GRACE_PERIOD='15s')
COMPOSE=['./scripts/compose.sh']
BASE='http://localhost:18082'
RUN_ID=uuid.uuid4().hex
def save(name,value):
temporary=OUT/(name+'.json.tmp')
temporary.write_text(json.dumps(value,indent=2)+'\n')
temporary.replace(OUT/(name+'.json'))
def stage(name):
save('acceptance',{'passed':False,'run_id':RUN_ID,'stage':name})
print(name,flush=True)
def command(args, name=None, expected=0, **kw):
p=subprocess.run(args,cwd=ROOT,env=ENV,text=True,capture_output=True,timeout=kw.pop('timeout',240),**kw)
if name:
(OUT/(name+'.txt')).write_text('$ '+' '.join(args)+'\n'+p.stdout+p.stderr)
check(p.returncode==expected,'command failed: '+' '.join(args)+' (see evidence/'+str(name)+'.txt)')
return p.stdout
def compose(*args,**kw): return command(COMPOSE+list(args),**kw)
def operator(*args,grace=None):
options=['-e','RECONCILE_GRACE_PERIOD='+grace] if grace else []
output=compose('run','--rm','-T',*options,'operator','storage',*args)
return json.loads(output)
def sql(query): return compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',query).strip()
def snapshot():
ids=compose('ps','-q','api','postgres','redis','s3','lb').split()
results=[]
for cid in ids:
info=json.loads(command(['docker','inspect','--format','{{json .State}}',cid]))
name=command(['docker','inspect','--format','{{.Name}}',cid]).strip().lstrip('/')
results.append({'name':name,'id':cid,'started_at':info['StartedAt'],'running':info['Running'],'health':info.get('Health',{}).get('Status')})
return results
def ready(c):
deadline=time.monotonic()+90
while time.monotonic()<deadline:
try: c.call('GET','/health/ready',schema='Health'); return
except Exception: time.sleep(.5)
raise AssertionError('API did not recover')
def padded_png(data,size):
padding=b'\0'*(size-len(data)-12)
chunk=b'paDd'+padding
return data[:-12]+struct.pack('>I',len(padding))+chunk+struct.pack('>I',zlib.crc32(chunk))+data[-12:]
def upload(c,actor,chat,data):
return c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',actor=actor,raw=data,expected=(201,429),headers={'Content-Type':'image/png'},schema='Attachment')
def direct_api(container,path,actor=None,body=b'',content_type='image/png'):
# Feed credentials via stdin, never argv or evidence. Only controlled results
# are returned to the report, not the raw HTTP request.
method='POST' if body else 'GET'
headers=[f'{method} {path} HTTP/1.1','Host: api','Connection: close']
if actor: headers.append('Authorization: Bearer '+actor['token'])
if body: headers.extend(['Content-Type: '+content_type,'Content-Length: '+str(len(body))])
raw=('\r\n'.join(headers)+'\r\n\r\n').encode()+body
p=subprocess.Popen(['docker','exec','-i',container,'nc','-w','10','127.0.0.1','8080'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
try:
# BusyBox nc closes the socket on stdin EOF. Keep stdin open until the
# response starts so the probe itself does not cancel the server context.
p.stdin.write(raw);p.stdin.flush()
readable,_,_=select.select([p.stdout],[],[],15)
check(bool(readable),'internal API probe timed out')
line=p.stdout.readline()
return int(line.split(b' ',2)[1])
finally:
p.stdin.close()
try: p.wait(timeout=3)
except subprocess.TimeoutExpired: p.kill();p.wait()
def failures(c,actor,chat,data):
findings={}
att=upload(c,actor,chat,data)['json']
c.message(actor,chat,'',attachments=[att['id']])
key=sql("SELECT object_key FROM attachments WHERE id='"+att['id']+"'")
raw=compose('exec','-T','api','/usr/local/bin/messenger','storage','private-check','--key',key)
findings['private_bucket']=json.loads(raw)
save('failures',findings)
before=snapshot()
command([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh','--base-url',BASE],name='persistence')
after=snapshot()
after_by_name={x['name']:x for x in after}
check(all(old['name'] in after_by_name and after_by_name[old['name']]['running'] and
(after_by_name[old['name']]['id']!=old['id'] or after_by_name[old['name']]['started_at']!=old['started_at'])
for old in before),'restart hook did not restart or replace every store/API/LB')
findings['restart']={'before':before,'after':after}
save('failures',findings)
findings['failover']=[]
for target in compose('ps','-q','api').split():
hostname=command(['docker','inspect','--format','{{.Config.Hostname}}',target]).strip()
for _ in range(20):
response=upload(c,actor,chat,data)
if response['headers']['X-Instance-Id'].startswith(hostname+'-'):break
else: raise AssertionError('cannot route upload to intended replica')
attachment=response['json'];alias=response['headers']['X-Instance-Id']
stop_env=dict(ENV,TARGET_INSTANCE_ID=alias)
p=subprocess.run(['./scripts/stop-one.sh'],cwd=ROOT,env=stop_env,capture_output=True,timeout=15)
check(p.returncode==0,'targeted stop failed')
deadline=time.monotonic()+10
while True:
try:
downloaded=c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor)
check(downloaded['bytes']==data,'failover changed original')
check(downloaded['headers']['X-Instance-Id']!=alias,'stopped uploader served download')
break
except Exception:
check(time.monotonic()<deadline,'download did not recover');time.sleep(.3)
check(command(['docker','inspect','--format','{{.State.Running}}',target]).strip()=='false','target not stopped')
findings['failover'].append({'stopped_container':target,'upload_instance':alias,'download_instance':downloaded['headers']['X-Instance-Id'],'sha256':hashlib.sha256(data).hexdigest()})
save('failures',findings)
compose('up','-d','--no-deps','--wait','api');ready(c)
command([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh','--base-url',BASE],name='failover')
compose('up','-d','--no-deps','--wait','api');ready(c)
api=compose('ps','-q','api').split()[0]
try:
compose('stop','s3')
check(direct_api(api,'/health/live')==200,'S3 outage killed liveness')
check(direct_api(api,'/health/ready')==503,'S3 outage did not fail readiness')
status=direct_api(api,'/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',actor,data)
check(status==503,'S3 failure upload must return 503')
blocked=compose('run','--rm','-T','operator','storage','reconcile','--apply',expected=1,name='reconcile-s3-down')
scan=json.loads(blocked)
check(scan['deleted']==0 and scan['marked_missing']==0,'S3 outage caused reconciliation writes')
findings['s3_outage']={'live':200,'ready':503,'upload':status,'reconciler':scan}
save('failures',findings)
finally:
compose('up','-d','--no-deps','--wait','s3')
compose('run','--rm','-T','s3-init')
ready(c)
# A client disconnect before all bytes are received cannot create a ready row.
count_before=sql('SELECT count(*) FROM attachments')
conn=http.client.HTTPConnection('localhost',18082,timeout=10)
conn.putrequest('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png')
conn.putheader('Authorization','Bearer '+actor['token']);conn.putheader('Content-Type','image/png');conn.putheader('Content-Length',len(data)*2);conn.endheaders();conn.send(data[:100]);conn.close();time.sleep(.5)
check(sql('SELECT count(*) FROM attachments')==count_before,'interrupted body created metadata')
findings['interrupted_upload']={'metadata_unchanged':True}
save('failures',findings)
# Real database-side timeout after S3 PUT, without production fault switches.
lock=subprocess.Popen(COMPOSE+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',"BEGIN; LOCK TABLE attachments IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(4); COMMIT;"],cwd=ROOT,env=ENV,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
try:
deadline=time.monotonic()+3
while sql("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE relation='attachments'::regclass AND mode='AccessExclusiveLock' AND granted)")!='t':
check(lock.poll() is None and time.monotonic()<deadline,'lock session failed')
time.sleep(.05)
failed=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',503,actor=actor,raw=data,headers={'Content-Type':'image/png'})
check(failed['status']==503,'database timeout not reported')
finally:
lock.communicate(timeout=10)
check(sql('SELECT count(*) FROM attachments')==count_before,'failed metadata write committed')
findings['s3_put_db_timeout']={'upload_status':503,'metadata_unchanged':True}
save('failures',findings)
findings['dry_run_after_failed_commit']=operator('reconcile')
check(findings['dry_run_after_failed_commit']['orphans']>=1,'failed DB write did not leave detectable orphan')
# Missing objects retain IDs/references and can only be restored from originals.
operator('remove-object','--id',att['id'])
dry=operator('reconcile');check(dry['missing']>=1,'missing object not detected')
apply=operator('reconcile','--apply');check(apply['missing']>=1,'missing object not marked')
c.call('GET','/api/v1/attachments/'+att['id'],503,actor=actor)
bad=compose('run','--rm','-T','operator','storage','restore','--id',att['id'],'--source','/fixtures/README.md',expected=1)
operator('restore','--id',att['id'],'--source','/fixtures/sample.png')
operator('restore','--id',att['id'],'--source','/fixtures/sample.png')
downloaded=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)
check(downloaded['bytes']==data,'restored bytes changed')
history=c.call('GET','/api/v1/chats/'+chat['id']+'/messages',actor=actor,schema='MessagePage')['json']
check(any(att['id'] in m['attachment_ids'] for m in history['items']),'restoration lost message reference')
findings['missing_repair']={'dry_run':dry,'apply':apply,'wrong_original_rejected':True,'restored_sha256':hashlib.sha256(data).hexdigest(),'reference_preserved':True,'repeat_safe':True}
save('failures',findings)
count_before=sql('SELECT count(*) FROM attachments')
try:
compose('stop','postgres')
raw=compose('run','--rm','-T','operator','storage','reconcile','--apply',expected=1,name='reconcile-db-down')
scan=json.loads(raw)
check(scan['deleted']==0 and scan['marked_missing']==0,'DB outage caused reconciliation writes')
findings['database_outage']={'reconciler':scan}
finally:
compose('up','-d','--no-deps','--wait','postgres');ready(c)
check(sql('SELECT count(*) FROM attachments')==count_before,'DB restart lost attachment metadata')
save('failures',findings)
return findings
def measurements(c,actor,chat,data):
records=[]
for size in [len(data),5*1024*1024]:
body=data if size==len(data) else padded_png(data,size)
for trial in range(1,4):
compose('up','-d','--no-deps','--force-recreate','--wait','api');ready(c)
upload(c,actor,chat,data) # One documented warm-up upload.
ids=compose('ps','-q','api').split()
cpu=subprocess.Popen(['docker','stats','--no-stream','--format','{{json .}}',*ids],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
barrier=threading.Barrier(10)
def work(_): barrier.wait();return upload(Client(BASE,timeout=30),actor,chat,body)
start=time.monotonic()
with futures.ThreadPoolExecutor(max_workers=10) as pool: results=list(pool.map(work,range(10)))
duration=time.monotonic()-start
successful=[x['elapsed'] for x in results if x['status']==201]
rss=[]
for cid in ids:
values=command(['docker','exec',cid,'sh','-c','cat /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.peak']).split()
rss.append({'container':cid,'memory_current_bytes':int(values[0]),'memory_peak_bytes':int(values[1])})
stats,_=cpu.communicate(timeout=15)
cpu_samples=[{'container':v.get('Name'),'cpu_percent':v.get('CPUPerc')} for v in [json.loads(line) for line in stats.splitlines()]]
def quantile(p): return sorted(successful)[min(len(successful)-1,int((len(successful)-1)*p))] if successful else None
records.append({'bytes':len(body),'trial':trial,'clients':10,'warmup_uploads':1,'duration_seconds':duration,'successful':len(successful),'errors':10-len(successful),'error_rate':(10-len(successful))/10,'successful_rps':len(successful)/duration,'p50':quantile(.5),'p95':quantile(.95),'p99':quantile(.99),'status_counts':{str(code):sum(x['status']==code for x in results) for code in [201,429]},'memory':rss,'cpu':cpu_samples})
save('uploads',records)
return records
def research(c,actor,chat,data):
results=[]
# Demonstrate freshness independently of the duration of mass seeding/scanning.
fresh_key=operator('seed')['keys'][0]
invalid_key='originals/not-a-server-uuid-'+uuid.uuid4().hex
outside_key='unmanaged/'+uuid.uuid4().hex
operator('seed','--key',invalid_key)
operator('seed','--key',outside_key)
fresh=operator('reconcile','--apply',grace='1h')
check(operator('object-check','--key',fresh_key)['exists'],'fresh object deleted')
# Upload and reconciliation execute concurrently; every acknowledged upload
# must remain downloadable. The separate PostgreSQL test checks lock contention.
with futures.ThreadPoolExecutor(max_workers=2) as workers:
scan=workers.submit(operator,'reconcile','--apply')
concurrent_upload=upload(c,actor,chat,data)
scan_result=scan.result()
check(concurrent_upload['status']==201,'concurrent upload failed')
att=concurrent_upload['json']
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)['bytes']==data,'concurrent reconciliation damaged upload')
safety={'fresh_apply':fresh,'fresh_key_preserved':True,'concurrent_scan':scan_result,'concurrent_upload_sha256':hashlib.sha256(data).hexdigest()}
save('reconciliation-safety',safety)
for count in [100,1000]:
operator('seed','--count',str(count))
# All objects exist before the grace wait starts. Duration of seeding is
# intentionally excluded from the scan timing, but dataset size is recorded.
time.sleep(16)
scans=[operator('reconcile') for _ in range(3)]
applied=operator('reconcile','--apply')
repeated=operator('reconcile','--apply')
check(applied['deleted']>=count,'old orphans not cleaned')
check(repeated['deleted']==0 and repeated['missing']==0,'repeat was not safe or valid original deleted')
check(operator('object-check','--key',invalid_key)['exists'],'invalid managed key deleted')
check(operator('object-check','--key',outside_key)['exists'],'object outside prefix deleted')
results.append({'seeded':count,'dry_runs':scans,'apply':applied,'repeat_apply':repeated,'invalid_key_preserved':True,'outside_prefix_preserved':True})
save('reconciliation',results)
return results
def main():
OUT.mkdir(exist_ok=True)
stage("bootstrap")
command(['./scripts/bootstrap.sh'],name='bootstrap')
before_keys=hashlib.sha256((ROOT/'secrets/lab4/s3.json').read_bytes()).hexdigest()
command(['./scripts/bootstrap.sh'],name='bootstrap-repeat')
check(before_keys==hashlib.sha256((ROOT/'secrets/lab4/s3.json').read_bytes()).hexdigest(),'bootstrap rotated credentials')
compose('config','--quiet',name='compose-config')
compose('up','-d','--build','--wait','--wait-timeout','120',name='stand-up',timeout=600)
meta={'run_id':RUN_ID,'date_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'git_head':command(['git','rev-parse','HEAD']).strip(),'working_tree_dirty':bool(command(['git','status','--porcelain']).strip()),'host':platform.platform(),'docker':json.loads(command(['docker','info','--format','{{json .}}'])),'project':ENV['COMPOSE_PROJECT_NAME'],'limits':{'api_replicas':2,'api_cpu_each':1,'api_ram_each':536870912,'upload_timeout':'5s','grace':'15s','upload_slots_each':4,'decode_slots_each':1}}
# Keep a small allow-list; docker info can contain unrelated host settings.
meta['docker']={k:meta['docker'][k] for k in ['ServerVersion','Architecture','NCPU','MemTotal']}
(OUT/'environment.json').write_text(json.dumps(meta,indent=2)+'\n')
compose('run','--rm','-T','s3-init',name='bucket-bootstrap-repeat')
c=Client(BASE,timeout=20);ready(c)
command(['make','check','build','vet'],name='build-checks')
validator=str(ROOT/'.venv/bin/python') if (ROOT/'.venv/bin/python').exists() else sys.executable
command(['make','validate','PYTHON='+validator],name='validate')
command(['make','test-go','BASE_URL='+BASE],name='go-tests')
command(['make','test-storage'],name='storage-tests')
command(['make','test','BASE_URL='+BASE],name='smoke')
actor,bob,eve,chat=c.bootstrap();data=(ROOT/'tests/fixtures/sample.png').read_bytes()
stage('faults')
findings=failures(c,actor,chat,data)
(OUT/'failures.json').write_text(json.dumps(findings,indent=2)+'\n')
stage('uploads')
(OUT/'uploads.json').write_text(json.dumps(measurements(c,actor,chat,data),indent=2)+'\n')
stage('reconciliation')
(OUT/'reconciliation.json').write_text(json.dumps(research(c,actor,chat,data),indent=2)+'\n')
result={'run_id':RUN_ID,'passed':True,'checks':['contract','go_race','transaction_rollback','smoke','private_bucket','persistence','both_uploaders_failover','public_failover','s3_down','interrupted_body','s3_put_db_timeout','missing_restore','10_client_memory','reconciler_grace','reconciler_repeat'],'remaining_metadata':int(sql('SELECT count(*) FROM attachments')),'timestamp_utc':datetime.datetime.now(datetime.timezone.utc).isoformat()}
compose('down',name='stand-down')
save('acceptance',result) # Preserve all experiment volumes.
print('PASS: lab4 acceptance; summaries in evidence/. Isolated stand stopped; volumes preserved.',flush=True)
if __name__=='__main__': main()