forked from mtusi2026/mtusi2026lab5
252 lines
12 KiB
Python
252 lines
12 KiB
Python
"""Explicit fault-injection campaign, scoped to the mtusi-lab5 Compose project."""
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
os.environ['PYTHONDONTWRITEBYTECODE']='1'
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0,str(ROOT/'tests'))
|
|
from client import Client,Failure,check
|
|
EVIDENCE = ROOT/'evidence'
|
|
BASE = os.getenv('BASE_URL','http://127.0.0.1:8085')
|
|
DC = ['docker','compose','--env-file','.env','-f','compose.yaml','-f','compose.test.yaml']
|
|
results = []
|
|
|
|
|
|
def command(args,filename=None,allow_failure=False):
|
|
proc = subprocess.run(args,cwd=ROOT,encoding='utf-8',errors='replace',capture_output=True,timeout=180)
|
|
if filename:
|
|
(EVIDENCE/filename).write_text(proc.stdout+proc.stderr,encoding='utf-8')
|
|
if proc.returncode and not allow_failure:
|
|
raise RuntimeError('Command failed: '+' '.join(args)+'\n'+proc.stdout+proc.stderr)
|
|
return proc
|
|
|
|
|
|
def compose(*args):
|
|
return command(DC+list(args))
|
|
|
|
|
|
def dbtool(*args,allow_failure=False):
|
|
proc = command(DC+['run','--rm','--no-deps','test-runner','python','student-tests/dbtool.py',*map(str,args)],allow_failure=allow_failure)
|
|
if allow_failure:
|
|
return proc.returncode
|
|
return json.loads(proc.stdout)
|
|
|
|
|
|
def save(name,value):
|
|
(EVIDENCE/name).write_text(json.dumps(value,ensure_ascii=False,indent=2),encoding='utf-8')
|
|
|
|
|
|
def wait_health(c):
|
|
deadline = time.monotonic()+90
|
|
while time.monotonic()<deadline:
|
|
try:
|
|
c.call('GET','/health/ready',schema='Health')
|
|
return
|
|
except Failure:
|
|
time.sleep(0.3)
|
|
raise AssertionError('API did not recover')
|
|
|
|
|
|
def snapshot():
|
|
ids = compose('ps','-q').stdout.split()
|
|
info = json.loads(command(['docker','inspect',*ids]).stdout)
|
|
return {x['Config']['Labels']['com.docker.compose.service']:
|
|
{'id':x['Id'],'started_at':x['State']['StartedAt'],
|
|
'image':x['Config']['Image'],'memory_limit':x['HostConfig']['Memory'],
|
|
'cpu_limit':x['HostConfig']['NanoCpus']} for x in info}
|
|
|
|
|
|
def record(name,started,value):
|
|
entry = {'scenario':name,'result':'PASS','duration_seconds':round(time.monotonic()-started,3),**value}
|
|
results.append(entry)
|
|
save('pipeline-results.json',{'date':dt.datetime.now(dt.timezone.utc).isoformat(),'scenarios':results})
|
|
print('PASS:',name,'seconds=',entry['duration_seconds'],flush=True)
|
|
|
|
|
|
def ready(c,actor,att,original):
|
|
metadata = c.wait_ready(actor,att)
|
|
content = c.call('GET',f"/api/v1/attachments/{att['id']}/content",actor=actor)['bytes']
|
|
check(content==original,'acknowledged original lost/changed')
|
|
info = dbtool('inspect',att['id'])
|
|
current = [x for x in info['objects'] if f"/v{info['version']}/" in x['key']]
|
|
check(len(current)==4,'duplicate or incomplete effect set')
|
|
check(len([j for j in info['jobs'] if j['status']=='ready'])==1,'multiple current ready jobs')
|
|
return metadata,info
|
|
|
|
|
|
def main():
|
|
EVIDENCE.mkdir(exist_ok=True)
|
|
command([sys.executable,'-X','utf8','scripts/check_contract.py'],'check.txt')
|
|
compose('config','--quiet')
|
|
compose('up','-d','--no-build')
|
|
c = Client(BASE,timeout=5)
|
|
wait_health(c)
|
|
command([sys.executable,'-X','utf8','tests/smoke.py','--base-url',BASE],'smoke.txt')
|
|
print('PASS: public smoke',flush=True)
|
|
proc = compose('run','--rm','--no-deps','test-runner','python','student-tests/pipeline.py')
|
|
(EVIDENCE/'acceptance.txt').write_text(proc.stdout,encoding='utf-8')
|
|
print('PASS: image and API acceptance',flush=True)
|
|
alice,bob,eve,chat = c.bootstrap()
|
|
try:
|
|
started = time.monotonic()
|
|
compose('stop','-t','1','worker')
|
|
att,data = c.attachment(alice,chat)
|
|
c.call('GET',f"/api/v1/attachments/{att['id']}/content",409,actor=alice)
|
|
message = c.message(alice,chat,'',attachments=[att['id']])['json']
|
|
check(message['attachment_ids']==[att['id']],'queued link missing')
|
|
deadline = time.monotonic()+10
|
|
while True:
|
|
queue_before = dbtool('queue')
|
|
if queue_before['length']>0:
|
|
break
|
|
check(time.monotonic()<deadline,'not published')
|
|
check(queue_before['aof']['appendonly']=='yes' and queue_before['fsync']['appendfsync']=='always','broker durability configuration')
|
|
before = snapshot()
|
|
compose('restart','api1','api2','broker')
|
|
wait_health(c)
|
|
after = snapshot()
|
|
for role in ('api1','api2','broker'):
|
|
check(before[role]['started_at']!=after[role]['started_at'],'restart did not happen')
|
|
queue_after = dbtool('queue')
|
|
check(queue_after['length']>0,'broker lost unconsumed job')
|
|
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
|
|
compose('start','worker')
|
|
_,info = ready(c,bob,att,data)
|
|
record('worker_off_api_broker_restart',started,{'queue_before':queue_before,'queue_after':queue_after,'before':before,'after':after,'job':info})
|
|
|
|
started = time.monotonic()
|
|
baseline = info
|
|
dbtool('duplicate',att['id'],1,12)
|
|
time.sleep(2)
|
|
repeated = dbtool('inspect',att['id'])
|
|
check(repeated['objects']==baseline['objects'],'successful duplicate rewrote objects')
|
|
check(repeated['jobs']==baseline['jobs'],'successful duplicate changed job')
|
|
check(any(e['event']=='duplicate_ignored' for e in repeated['events']),'duplicates not consumed')
|
|
record('12_duplicate_deliveries',started,{'job':repeated})
|
|
|
|
for point in ('before_publish','after_publish','before_effect','after_object','before_result_commit','after_result_commit'):
|
|
started = time.monotonic()
|
|
compose('stop','-t','1','worker','publisher')
|
|
att,data = c.attachment(alice,chat)
|
|
dbtool('fault',att['id'],point)
|
|
compose('start','publisher','worker')
|
|
_,info = ready(c,bob,att,data)
|
|
check(any(e['event']=='fault' and e['detail']==point for e in info['events']),'fault not triggered')
|
|
if point in ('before_effect','after_object','before_result_commit'):
|
|
check(any(e['event']=='lease_recovered' for e in info['events']),'dead worker lease not recovered')
|
|
check(info['jobs'][0]['attempts']==2,'crash attempt accounting')
|
|
if point=='after_result_commit':
|
|
deadline = time.monotonic()+40
|
|
while not any(e['event']=='duplicate_ignored' for e in info['events']):
|
|
check(time.monotonic()<deadline,'committed crash did not redeliver')
|
|
time.sleep(0.5)
|
|
info = dbtool('inspect',att['id'])
|
|
check(len(info['objects'])==4,'duplicate effect set after crash')
|
|
record(point,started,{'job':info})
|
|
|
|
started = time.monotonic()
|
|
compose('stop','-t','1','worker')
|
|
att,data = c.attachment(alice,chat)
|
|
dbtool('fault',att['id'],'transient',2)
|
|
compose('start','worker')
|
|
_,info = ready(c,bob,att,data)
|
|
delays = [float(e['detail']) for e in info['events'] if e['event']=='retry_scheduled']
|
|
check(info['jobs'][0]['attempts']==3 and len(delays)==2,'retry accounting')
|
|
check(0.1<=delays[0]<=0.3 and 0.2<=delays[1]<=0.6,'exponential backoff/jitter')
|
|
record('transient_backoff_jitter',started,{'job':info,'delays':delays})
|
|
|
|
started = time.monotonic()
|
|
compose('stop','-t','1','worker')
|
|
att,data = c.attachment(alice,chat)
|
|
dbtool('fault',att['id'],'hang')
|
|
compose('start','worker')
|
|
_,info = ready(c,bob,att,data)
|
|
check(info['jobs'][0]['attempts']==2,'wall-clock timeout did not retry')
|
|
record('real_job_timeout',started,{'job':info})
|
|
|
|
for fault,count,expected,attempts in [('permanent',1,'invalid_image',1),('transient',5,'retry_exhausted',5)]:
|
|
started = time.monotonic()
|
|
compose('stop','-t','1','worker')
|
|
att,data = c.attachment(alice,chat)
|
|
dbtool('fault',att['id'],fault,count)
|
|
compose('start','worker')
|
|
deadline = time.monotonic()+60
|
|
while True:
|
|
metadata = c.call('GET',f"/api/v1/attachments/{att['id']}",actor=alice,schema='Attachment')['json']
|
|
if metadata['status']=='failed':
|
|
break
|
|
check(time.monotonic()<deadline,'failed deadline')
|
|
time.sleep(0.2)
|
|
check(metadata['error_code']==expected,'unsafe/incorrect error code')
|
|
c.call('GET',f"/api/v1/attachments/{att['id']}/content",409,actor=bob)
|
|
c.call('POST',f"/api/v1/chats/{chat['id']}/messages",409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':'failed-'+att['id']})
|
|
failed = dbtool('inspect',att['id'])
|
|
check(failed['jobs'][0]['attempts']==attempts,'bounded attempts')
|
|
dbtool('replay',att['id'])
|
|
_,info = ready(c,bob,att,data)
|
|
check(any(e['event']=='operator_replay' for e in info['events']),'replay audit')
|
|
check(dbtool('replay',att['id'],allow_failure=True)!=0,'replay of ready accepted')
|
|
record(fault+'_failed_replay',started,{'failed':failed,'recovered':info})
|
|
|
|
started = time.monotonic()
|
|
att,data = c.attachment(alice,chat)
|
|
_,v1 = ready(c,bob,att,data)
|
|
compose('stop','-t','1','worker')
|
|
dbtool('upgrade',att['id'])
|
|
# Old jobs arrive before, around, and after the new-version notification.
|
|
dbtool('duplicate',att['id'],1,3)
|
|
compose('start','worker')
|
|
_,v2 = ready(c,bob,att,data)
|
|
check(v2['version']==2 and len(v2['objects'])==8,'versioned keys/results')
|
|
dbtool('duplicate',att['id'],1,3)
|
|
dbtool('duplicate',att['id'],2,6)
|
|
time.sleep(2)
|
|
final = dbtool('inspect',att['id'])
|
|
check(final['objects']==v2['objects'] and final['version']==2 and final['status']=='ready','stale job overwrote new result')
|
|
check(any(e['event']=='stale_ignored' for e in final['events']),'stale delivery not observed')
|
|
record('pipeline_v2_reordered_stale_jobs',started,{'v1':v1,'v2':v2,'final':final})
|
|
|
|
started = time.monotonic()
|
|
compose('stop','-t','1','broker')
|
|
att,data = c.attachment(alice,chat)
|
|
c.message(alice,chat,'broker offline',attachments=[att['id']])
|
|
c.call('GET','/health/ready',schema='Health')
|
|
compose('restart','api1','api2')
|
|
wait_health(c)
|
|
compose('start','broker')
|
|
_,info = ready(c,bob,att,data)
|
|
record('broker_offline_outbox_recovery',started,{'job':info})
|
|
|
|
started = time.monotonic()
|
|
dbtool('invalid')
|
|
time.sleep(2)
|
|
queue = dbtool('queue')
|
|
check(queue['invalid']>=1,'invalid envelope registry')
|
|
stages = dbtool('staging')
|
|
check('staging/'+att['id'] not in stages['keys'],'ready staging not cleaned')
|
|
record('invalid_envelope_and_staging_cleanup',started,{'queue':queue,'staging_count':len(stages['keys'])})
|
|
finally:
|
|
compose('start','api1','api2','broker','s3','postgres','publisher','worker')
|
|
save('before-restart.json',snapshot())
|
|
suffix = '.cmd' if os.name=='nt' else '.sh'
|
|
command([sys.executable,'-X','utf8','tests/resilience.py','--base-url',BASE,'--mode','persistence','--action-script',str(ROOT/'scripts'/('restart'+suffix))],'persistence.txt')
|
|
save('after-restart.json',snapshot())
|
|
print('PASS: public persistence',flush=True)
|
|
command([sys.executable,'-X','utf8','tests/resilience.py','--base-url',BASE,'--mode','failover','--action-script',str(ROOT/'scripts'/('stop-one'+suffix))],'failover.txt')
|
|
print('PASS: public failover',flush=True)
|
|
compose('start','api1','api2')
|
|
wait_health(c)
|
|
command([sys.executable,'-X','utf8','tests/smoke.py','--base-url',BASE],'final-smoke.txt')
|
|
save('final-containers.json',snapshot())
|
|
save('verification.json',{'result':'PASS','pipeline_scenarios':len(results),'public_smoke':True,'public_persistence':True,'public_failover':True,'image_acceptance':True,'base_url':BASE})
|
|
print('PASS: all lab5 verification',flush=True)
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|