Files
mtusi2026lab5/scripts/verify.py
T

252 lines
12 KiB
Python

"""Explicit fault-injection campaign, scoped to the mtusi-lab5 Compose project."""
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
os.environ['PYTHONDONTWRITEBYTECODE']='1'
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client,Failure,check
EVIDENCE = ROOT/'evidence'
BASE = os.getenv('BASE_URL','http://127.0.0.1:8085')
DC = ['docker','compose','--env-file','.env','-f','compose.yaml','-f','compose.test.yaml']
results = []
def command(args,filename=None,allow_failure=False):
proc = subprocess.run(args,cwd=ROOT,encoding='utf-8',errors='replace',capture_output=True,timeout=180)
if filename:
(EVIDENCE/filename).write_text(proc.stdout+proc.stderr,encoding='utf-8')
if proc.returncode and not allow_failure:
raise RuntimeError('Command failed: '+' '.join(args)+'\n'+proc.stdout+proc.stderr)
return proc
def compose(*args):
return command(DC+list(args))
def dbtool(*args,allow_failure=False):
proc = command(DC+['run','--rm','--no-deps','test-runner','python','student-tests/dbtool.py',*map(str,args)],allow_failure=allow_failure)
if allow_failure:
return proc.returncode
return json.loads(proc.stdout)
def save(name,value):
(EVIDENCE/name).write_text(json.dumps(value,ensure_ascii=False,indent=2),encoding='utf-8')
def wait_health(c):
deadline = time.monotonic()+90
while time.monotonic()<deadline:
try:
c.call('GET','/health/ready',schema='Health')
return
except Failure:
time.sleep(0.3)
raise AssertionError('API did not recover')
def snapshot():
ids = compose('ps','-q').stdout.split()
info = json.loads(command(['docker','inspect',*ids]).stdout)
return {x['Config']['Labels']['com.docker.compose.service']:
{'id':x['Id'],'started_at':x['State']['StartedAt'],
'image':x['Config']['Image'],'memory_limit':x['HostConfig']['Memory'],
'cpu_limit':x['HostConfig']['NanoCpus']} for x in info}
def record(name,started,value):
entry = {'scenario':name,'result':'PASS','duration_seconds':round(time.monotonic()-started,3),**value}
results.append(entry)
save('pipeline-results.json',{'date':dt.datetime.now(dt.timezone.utc).isoformat(),'scenarios':results})
print('PASS:',name,'seconds=',entry['duration_seconds'],flush=True)
def ready(c,actor,att,original):
metadata = c.wait_ready(actor,att)
content = c.call('GET',f"/api/v1/attachments/{att['id']}/content",actor=actor)['bytes']
check(content==original,'acknowledged original lost/changed')
info = dbtool('inspect',att['id'])
current = [x for x in info['objects'] if f"/v{info['version']}/" in x['key']]
check(len(current)==4,'duplicate or incomplete effect set')
check(len([j for j in info['jobs'] if j['status']=='ready'])==1,'multiple current ready jobs')
return metadata,info
def main():
EVIDENCE.mkdir(exist_ok=True)
command([sys.executable,'-X','utf8','scripts/check_contract.py'],'check.txt')
compose('config','--quiet')
compose('up','-d','--no-build')
c = Client(BASE,timeout=5)
wait_health(c)
command([sys.executable,'-X','utf8','tests/smoke.py','--base-url',BASE],'smoke.txt')
print('PASS: public smoke',flush=True)
proc = compose('run','--rm','--no-deps','test-runner','python','student-tests/pipeline.py')
(EVIDENCE/'acceptance.txt').write_text(proc.stdout,encoding='utf-8')
print('PASS: image and API acceptance',flush=True)
alice,bob,eve,chat = c.bootstrap()
try:
started = time.monotonic()
compose('stop','-t','1','worker')
att,data = c.attachment(alice,chat)
c.call('GET',f"/api/v1/attachments/{att['id']}/content",409,actor=alice)
message = c.message(alice,chat,'',attachments=[att['id']])['json']
check(message['attachment_ids']==[att['id']],'queued link missing')
deadline = time.monotonic()+10
while True:
queue_before = dbtool('queue')
if queue_before['length']>0:
break
check(time.monotonic()<deadline,'not published')
check(queue_before['aof']['appendonly']=='yes' and queue_before['fsync']['appendfsync']=='always','broker durability configuration')
before = snapshot()
compose('restart','api1','api2','broker')
wait_health(c)
after = snapshot()
for role in ('api1','api2','broker'):
check(before[role]['started_at']!=after[role]['started_at'],'restart did not happen')
queue_after = dbtool('queue')
check(queue_after['length']>0,'broker lost unconsumed job')
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
compose('start','worker')
_,info = ready(c,bob,att,data)
record('worker_off_api_broker_restart',started,{'queue_before':queue_before,'queue_after':queue_after,'before':before,'after':after,'job':info})
started = time.monotonic()
baseline = info
dbtool('duplicate',att['id'],1,12)
time.sleep(2)
repeated = dbtool('inspect',att['id'])
check(repeated['objects']==baseline['objects'],'successful duplicate rewrote objects')
check(repeated['jobs']==baseline['jobs'],'successful duplicate changed job')
check(any(e['event']=='duplicate_ignored' for e in repeated['events']),'duplicates not consumed')
record('12_duplicate_deliveries',started,{'job':repeated})
for point in ('before_publish','after_publish','before_effect','after_object','before_result_commit','after_result_commit'):
started = time.monotonic()
compose('stop','-t','1','worker','publisher')
att,data = c.attachment(alice,chat)
dbtool('fault',att['id'],point)
compose('start','publisher','worker')
_,info = ready(c,bob,att,data)
check(any(e['event']=='fault' and e['detail']==point for e in info['events']),'fault not triggered')
if point in ('before_effect','after_object','before_result_commit'):
check(any(e['event']=='lease_recovered' for e in info['events']),'dead worker lease not recovered')
check(info['jobs'][0]['attempts']==2,'crash attempt accounting')
if point=='after_result_commit':
deadline = time.monotonic()+40
while not any(e['event']=='duplicate_ignored' for e in info['events']):
check(time.monotonic()<deadline,'committed crash did not redeliver')
time.sleep(0.5)
info = dbtool('inspect',att['id'])
check(len(info['objects'])==4,'duplicate effect set after crash')
record(point,started,{'job':info})
started = time.monotonic()
compose('stop','-t','1','worker')
att,data = c.attachment(alice,chat)
dbtool('fault',att['id'],'transient',2)
compose('start','worker')
_,info = ready(c,bob,att,data)
delays = [float(e['detail']) for e in info['events'] if e['event']=='retry_scheduled']
check(info['jobs'][0]['attempts']==3 and len(delays)==2,'retry accounting')
check(0.1<=delays[0]<=0.3 and 0.2<=delays[1]<=0.6,'exponential backoff/jitter')
record('transient_backoff_jitter',started,{'job':info,'delays':delays})
started = time.monotonic()
compose('stop','-t','1','worker')
att,data = c.attachment(alice,chat)
dbtool('fault',att['id'],'hang')
compose('start','worker')
_,info = ready(c,bob,att,data)
check(info['jobs'][0]['attempts']==2,'wall-clock timeout did not retry')
record('real_job_timeout',started,{'job':info})
for fault,count,expected,attempts in [('permanent',1,'invalid_image',1),('transient',5,'retry_exhausted',5)]:
started = time.monotonic()
compose('stop','-t','1','worker')
att,data = c.attachment(alice,chat)
dbtool('fault',att['id'],fault,count)
compose('start','worker')
deadline = time.monotonic()+60
while True:
metadata = c.call('GET',f"/api/v1/attachments/{att['id']}",actor=alice,schema='Attachment')['json']
if metadata['status']=='failed':
break
check(time.monotonic()<deadline,'failed deadline')
time.sleep(0.2)
check(metadata['error_code']==expected,'unsafe/incorrect error code')
c.call('GET',f"/api/v1/attachments/{att['id']}/content",409,actor=bob)
c.call('POST',f"/api/v1/chats/{chat['id']}/messages",409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':'failed-'+att['id']})
failed = dbtool('inspect',att['id'])
check(failed['jobs'][0]['attempts']==attempts,'bounded attempts')
dbtool('replay',att['id'])
_,info = ready(c,bob,att,data)
check(any(e['event']=='operator_replay' for e in info['events']),'replay audit')
check(dbtool('replay',att['id'],allow_failure=True)!=0,'replay of ready accepted')
record(fault+'_failed_replay',started,{'failed':failed,'recovered':info})
started = time.monotonic()
att,data = c.attachment(alice,chat)
_,v1 = ready(c,bob,att,data)
compose('stop','-t','1','worker')
dbtool('upgrade',att['id'])
# Old jobs arrive before, around, and after the new-version notification.
dbtool('duplicate',att['id'],1,3)
compose('start','worker')
_,v2 = ready(c,bob,att,data)
check(v2['version']==2 and len(v2['objects'])==8,'versioned keys/results')
dbtool('duplicate',att['id'],1,3)
dbtool('duplicate',att['id'],2,6)
time.sleep(2)
final = dbtool('inspect',att['id'])
check(final['objects']==v2['objects'] and final['version']==2 and final['status']=='ready','stale job overwrote new result')
check(any(e['event']=='stale_ignored' for e in final['events']),'stale delivery not observed')
record('pipeline_v2_reordered_stale_jobs',started,{'v1':v1,'v2':v2,'final':final})
started = time.monotonic()
compose('stop','-t','1','broker')
att,data = c.attachment(alice,chat)
c.message(alice,chat,'broker offline',attachments=[att['id']])
c.call('GET','/health/ready',schema='Health')
compose('restart','api1','api2')
wait_health(c)
compose('start','broker')
_,info = ready(c,bob,att,data)
record('broker_offline_outbox_recovery',started,{'job':info})
started = time.monotonic()
dbtool('invalid')
time.sleep(2)
queue = dbtool('queue')
check(queue['invalid']>=1,'invalid envelope registry')
stages = dbtool('staging')
check('staging/'+att['id'] not in stages['keys'],'ready staging not cleaned')
record('invalid_envelope_and_staging_cleanup',started,{'queue':queue,'staging_count':len(stages['keys'])})
finally:
compose('start','api1','api2','broker','s3','postgres','publisher','worker')
save('before-restart.json',snapshot())
suffix = '.cmd' if os.name=='nt' else '.sh'
command([sys.executable,'-X','utf8','tests/resilience.py','--base-url',BASE,'--mode','persistence','--action-script',str(ROOT/'scripts'/('restart'+suffix))],'persistence.txt')
save('after-restart.json',snapshot())
print('PASS: public persistence',flush=True)
command([sys.executable,'-X','utf8','tests/resilience.py','--base-url',BASE,'--mode','failover','--action-script',str(ROOT/'scripts'/('stop-one'+suffix))],'failover.txt')
print('PASS: public failover',flush=True)
compose('start','api1','api2')
wait_health(c)
command([sys.executable,'-X','utf8','tests/smoke.py','--base-url',BASE],'final-smoke.txt')
save('final-containers.json',snapshot())
save('verification.json',{'result':'PASS','pipeline_scenarios':len(results),'public_smoke':True,'public_persistence':True,'public_failover':True,'image_acceptance':True,'base_url':BASE})
print('PASS: all lab5 verification',flush=True)
if __name__=='__main__':
main()