Files
mtusi2026lab5/tests/pipeline_faults.py
T

144 lines
9.4 KiB
Python

"""Track 2: real crash processes, broker redelivery, version fencing and retries.
Run only with the normal worker and publisher stopped. Creates synthetic rows.
"""
import concurrent.futures
import hashlib
import io
import json
from pathlib import Path
import subprocess
import sys
import time
from PIL import Image
sys.path.insert(0,str(Path(__file__).resolve().parent))
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
from client import Client,check,validate
from app import store,object_store
from pipeline import broker,envelope,fetch_job,MAX_ATTEMPTS,replay,upgrade,STREAM
from worker import handle
from publisher import publish_once
def main():
store.pool.open(wait=True,timeout=20)
c=Client('http://lb:8080');alice,bob,eve,chat=c.bootstrap();r=broker()
rows=[]
def upload():
att,data=c.attachment(alice,chat)
with store.connect() as conn:
jid=str(conn.execute('SELECT id FROM jobs WHERE attachment_id=%s',(att['id'],)).fetchone()[0])
msg=envelope(fetch_job(conn,jid))
validate(msg,json.loads((Path(__file__).resolve().parents[1]/'contracts/jobs.schema.json').read_text()))
return att,data,jid,msg
def metadata(att):return c.call('GET','/api/v1/attachments/'+att['id'],actor=alice,schema='Attachment')['json']
def objects(att,version=1):
return sorted(x['Key'] for x in object_store.client.list_objects_v2(Bucket=object_store.bucket,Prefix=f"results/{att['id']}/v{version}/").get('Contents',[]))
def invariant(att,data,version=1):
ready=metadata(att);check(ready['status']=='ready','lost accepted work')
check(len(ready['variants'])==3,'wrong number of variants')
check(len(objects(att,version))==4,'duplicate or missing objects')
for name in ('original','thumbnail','crop','optimized'):
result=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+name,actor=bob)
if name=='original':check(result['bytes']==data,'original mutated')
else:
v=next(x for x in ready['variants'] if x['name']==name)
check(hashlib.sha256(result['bytes']).hexdigest()==v['sha256'],'variant checksum mismatch')
with Image.open(io.BytesIO(result['bytes'])) as im:
check((im.width,im.height)==(v['width'],v['height']),'false dimensions')
check(not im.getexif(),'EXIF not stripped')
return ready
try:
for iteration in range(3):
for crash in ('before_effect','after_effect','after_commit'):
att,data,jid,msg=upload();start=time.monotonic()
result=subprocess.run([sys.executable,'tests/drive_job.py',jid,'--crash',crash],capture_output=True,text=True)
check(result.returncode==86,'failpoint did not kill the worker process')
before=objects(att)
check(len(before)==(0 if crash=='before_effect' else 4),'wrong crash window')
with store.connect() as conn:
j=fetch_job(conn,jid)
if crash!='after_commit':
check(j['state']=='processing','lease not persisted')
check(handle(msg)=='defer','live lease was stolen')
# Controlled clock acceleration for matrix speed. Separate
# persistence test exercises real 30-second lease timeout.
conn.execute("UPDATE jobs SET lease_until=now()-interval '1 second' WHERE id=%s",(jid,))
result=subprocess.run([sys.executable,'tests/drive_job.py',jid,'--recover'],capture_output=True,text=True)
check(result.returncode==0,'redelivery process failed')
check(json.loads(result.stdout)['outcome'] in ('ready','noop'),'redelivery did not complete')
ready=invariant(att,data)
# Successful duplicate must not update metadata or re-encode.
check(handle(msg)=='noop','ready duplicate was re-executed')
check(metadata(att)==ready,'ready changed after duplicate')
with store.connect() as conn:attempts=fetch_job(conn,jid)['attempt']
rows.append(dict(iteration=iteration+1,scenario=crash,status='PASS',attempts=attempts,
recovery_ms=round((time.monotonic()-start)*1000,3),objects=4,lease_expiry='accelerated' if crash!='after_commit' else 'not_needed'))
att,data,jid,msg=upload();start=time.monotonic()
outcomes=[]
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
outcomes=list(pool.map(lambda _:handle(msg),range(8)))
invariant(att,data)
check(outcomes.count('ready')==1,'concurrent duplicate produced multiple effects')
rows.append(dict(iteration=iteration+1,scenario='concurrent_duplicates',status='PASS',deliveries=8,
effects=1,recovery_ms=round((time.monotonic()-start)*1000,3),outcomes=outcomes,objects=4))
# New version overtakes an unprocessed old version.
att,data,jid,old=upload();new_jid=upgrade(store,att['id'])
with store.connect() as conn:new=envelope(fetch_job(conn,new_jid))
start=time.monotonic();check(handle(new)=='ready','new version failed')
newest=invariant(att,data,2)
for _ in range(5):check(handle(old)=='noop','stale job not fenced')
check(metadata(att)==newest and not objects(att,1),'old version overwrote new')
rows.append(dict(iteration=iteration+1,scenario='reorder_stale_version',status='PASS',old_deliveries=5,
current_version=2,old_objects=0,objects=4,recovery_ms=round((time.monotonic()-start)*1000,3)))
# Upgrade an already published result, then redeliver the old version.
att,data,jid,old=upload();check(handle(old)=='ready','v1 failed')
v1=invariant(att,data)
new_jid=upgrade(store,att['id'])
with store.connect() as conn:new=envelope(fetch_job(conn,new_jid))
check(handle(new)=='ready','upgrade failed');v2=invariant(att,data,2)
check(v1['variants']!=v2['variants'],'version settings did not change result')
check(handle(old)=='noop' and metadata(att)==v2,'published old version replaced upgraded result')
rows.append(dict(scenario='published_v1_then_v2_then_stale_v1',status='PASS',current_version=2,objects=4))
# Permanent errors, explicit audited replay, ready replay refused.
att,data,jid,msg=upload();check(handle(msg,force_error='permanent')=='failed','permanent failure retried')
check(metadata(att)['status']=='failed','failed not exposed')
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':'failed-test'})
replay(store,jid);check(handle(msg)=='ready','operator replay did not recover')
invariant(att,data)
try:replay(store,jid);raise AssertionError('ready replay allowed')
except ValueError:pass
rows.append(dict(scenario='permanent_failed_replay',status='PASS',objects=4))
# Transient retry scheduling respects backoff and exhausts the bound.
att,data,jid,msg=upload();delays=[]
for attempt in range(MAX_ATTEMPTS):
check(handle(msg,force_error='temporary')=='retry','temporary failure not handled')
with store.connect() as conn:
j=fetch_job(conn,jid)
if attempt<MAX_ATTEMPTS-1:
delay=conn.execute('SELECT extract(epoch FROM next_at-now()) FROM jobs WHERE id=%s',(jid,)).fetchone()[0]
check(delay>0,'no backoff');delays.append(float(delay))
check(handle(msg)=='defer','retry ignored schedule')
conn.execute("UPDATE jobs SET next_at=now()-interval '1 second' WHERE id=%s",(jid,))
check(metadata(att)['error_code']=='retry_exhausted','retry was not bounded')
replay(store,jid);check(handle(msg)=='ready','retry exhausted replay failed');invariant(att,data)
rows.append(dict(scenario='bounded_retry_jitter',status='PASS',max_attempts=MAX_ATTEMPTS,delays_seconds=delays))
# COMMIT -> publish recovery, and a lost publish confirm duplicates safely.
att,data,jid,msg=upload()
with store.connect() as conn:check(conn.execute('SELECT published_at FROM outbox WHERE job_id=%s',(jid,)).fetchone()[0] is None,'outbox already published')
publish_once(r)
with store.connect() as conn:
conn.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(jid,))
publish_once(r)
check(handle(msg)=='ready' and handle(msg)=='noop','republish was unsafe');invariant(att,data)
rows.append(dict(scenario='outbox_republish',status='PASS',objects=4))
with store.connect() as conn:
accepted=conn.execute('SELECT count(*) FROM attachments').fetchone()[0]
result={'status':'PASS','track':2,'matrix_repeats':3,'rows':rows,'accepted_in_database':accepted,
'invariants':{'accepted_tasks_lost':0,'duplicate_variant_sets':0,'stale_version_overwrites':0},
'limitations':['matrix lease expiry accelerated with SQL','single host, worker processing is serial except intentional duplicate probe','no disk-loss or network partition proof']}
print(json.dumps(result,ensure_ascii=False,indent=2))
finally:r.close();store.pool.close();object_store.close()
if __name__=='__main__':main()