144 lines
9.4 KiB
Python
144 lines
9.4 KiB
Python
"""Track 2: real crash processes, broker redelivery, version fencing and retries.
|
|
Run only with the normal worker and publisher stopped. Creates synthetic rows.
|
|
"""
|
|
import concurrent.futures
|
|
import hashlib
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from PIL import Image
|
|
sys.path.insert(0,str(Path(__file__).resolve().parent))
|
|
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
|
from client import Client,check,validate
|
|
from app import store,object_store
|
|
from pipeline import broker,envelope,fetch_job,MAX_ATTEMPTS,replay,upgrade,STREAM
|
|
from worker import handle
|
|
from publisher import publish_once
|
|
|
|
|
|
def main():
|
|
store.pool.open(wait=True,timeout=20)
|
|
c=Client('http://lb:8080');alice,bob,eve,chat=c.bootstrap();r=broker()
|
|
rows=[]
|
|
def upload():
|
|
att,data=c.attachment(alice,chat)
|
|
with store.connect() as conn:
|
|
jid=str(conn.execute('SELECT id FROM jobs WHERE attachment_id=%s',(att['id'],)).fetchone()[0])
|
|
msg=envelope(fetch_job(conn,jid))
|
|
validate(msg,json.loads((Path(__file__).resolve().parents[1]/'contracts/jobs.schema.json').read_text()))
|
|
return att,data,jid,msg
|
|
def metadata(att):return c.call('GET','/api/v1/attachments/'+att['id'],actor=alice,schema='Attachment')['json']
|
|
def objects(att,version=1):
|
|
return sorted(x['Key'] for x in object_store.client.list_objects_v2(Bucket=object_store.bucket,Prefix=f"results/{att['id']}/v{version}/").get('Contents',[]))
|
|
def invariant(att,data,version=1):
|
|
ready=metadata(att);check(ready['status']=='ready','lost accepted work')
|
|
check(len(ready['variants'])==3,'wrong number of variants')
|
|
check(len(objects(att,version))==4,'duplicate or missing objects')
|
|
for name in ('original','thumbnail','crop','optimized'):
|
|
result=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+name,actor=bob)
|
|
if name=='original':check(result['bytes']==data,'original mutated')
|
|
else:
|
|
v=next(x for x in ready['variants'] if x['name']==name)
|
|
check(hashlib.sha256(result['bytes']).hexdigest()==v['sha256'],'variant checksum mismatch')
|
|
with Image.open(io.BytesIO(result['bytes'])) as im:
|
|
check((im.width,im.height)==(v['width'],v['height']),'false dimensions')
|
|
check(not im.getexif(),'EXIF not stripped')
|
|
return ready
|
|
try:
|
|
for iteration in range(3):
|
|
for crash in ('before_effect','after_effect','after_commit'):
|
|
att,data,jid,msg=upload();start=time.monotonic()
|
|
result=subprocess.run([sys.executable,'tests/drive_job.py',jid,'--crash',crash],capture_output=True,text=True)
|
|
check(result.returncode==86,'failpoint did not kill the worker process')
|
|
before=objects(att)
|
|
check(len(before)==(0 if crash=='before_effect' else 4),'wrong crash window')
|
|
with store.connect() as conn:
|
|
j=fetch_job(conn,jid)
|
|
if crash!='after_commit':
|
|
check(j['state']=='processing','lease not persisted')
|
|
check(handle(msg)=='defer','live lease was stolen')
|
|
# Controlled clock acceleration for matrix speed. Separate
|
|
# persistence test exercises real 30-second lease timeout.
|
|
conn.execute("UPDATE jobs SET lease_until=now()-interval '1 second' WHERE id=%s",(jid,))
|
|
result=subprocess.run([sys.executable,'tests/drive_job.py',jid,'--recover'],capture_output=True,text=True)
|
|
check(result.returncode==0,'redelivery process failed')
|
|
check(json.loads(result.stdout)['outcome'] in ('ready','noop'),'redelivery did not complete')
|
|
ready=invariant(att,data)
|
|
# Successful duplicate must not update metadata or re-encode.
|
|
check(handle(msg)=='noop','ready duplicate was re-executed')
|
|
check(metadata(att)==ready,'ready changed after duplicate')
|
|
with store.connect() as conn:attempts=fetch_job(conn,jid)['attempt']
|
|
rows.append(dict(iteration=iteration+1,scenario=crash,status='PASS',attempts=attempts,
|
|
recovery_ms=round((time.monotonic()-start)*1000,3),objects=4,lease_expiry='accelerated' if crash!='after_commit' else 'not_needed'))
|
|
att,data,jid,msg=upload();start=time.monotonic()
|
|
outcomes=[]
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
|
outcomes=list(pool.map(lambda _:handle(msg),range(8)))
|
|
invariant(att,data)
|
|
check(outcomes.count('ready')==1,'concurrent duplicate produced multiple effects')
|
|
rows.append(dict(iteration=iteration+1,scenario='concurrent_duplicates',status='PASS',deliveries=8,
|
|
effects=1,recovery_ms=round((time.monotonic()-start)*1000,3),outcomes=outcomes,objects=4))
|
|
# New version overtakes an unprocessed old version.
|
|
att,data,jid,old=upload();new_jid=upgrade(store,att['id'])
|
|
with store.connect() as conn:new=envelope(fetch_job(conn,new_jid))
|
|
start=time.monotonic();check(handle(new)=='ready','new version failed')
|
|
newest=invariant(att,data,2)
|
|
for _ in range(5):check(handle(old)=='noop','stale job not fenced')
|
|
check(metadata(att)==newest and not objects(att,1),'old version overwrote new')
|
|
rows.append(dict(iteration=iteration+1,scenario='reorder_stale_version',status='PASS',old_deliveries=5,
|
|
current_version=2,old_objects=0,objects=4,recovery_ms=round((time.monotonic()-start)*1000,3)))
|
|
# Upgrade an already published result, then redeliver the old version.
|
|
att,data,jid,old=upload();check(handle(old)=='ready','v1 failed')
|
|
v1=invariant(att,data)
|
|
new_jid=upgrade(store,att['id'])
|
|
with store.connect() as conn:new=envelope(fetch_job(conn,new_jid))
|
|
check(handle(new)=='ready','upgrade failed');v2=invariant(att,data,2)
|
|
check(v1['variants']!=v2['variants'],'version settings did not change result')
|
|
check(handle(old)=='noop' and metadata(att)==v2,'published old version replaced upgraded result')
|
|
rows.append(dict(scenario='published_v1_then_v2_then_stale_v1',status='PASS',current_version=2,objects=4))
|
|
# Permanent errors, explicit audited replay, ready replay refused.
|
|
att,data,jid,msg=upload();check(handle(msg,force_error='permanent')=='failed','permanent failure retried')
|
|
check(metadata(att)['status']=='failed','failed not exposed')
|
|
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':'failed-test'})
|
|
replay(store,jid);check(handle(msg)=='ready','operator replay did not recover')
|
|
invariant(att,data)
|
|
try:replay(store,jid);raise AssertionError('ready replay allowed')
|
|
except ValueError:pass
|
|
rows.append(dict(scenario='permanent_failed_replay',status='PASS',objects=4))
|
|
# Transient retry scheduling respects backoff and exhausts the bound.
|
|
att,data,jid,msg=upload();delays=[]
|
|
for attempt in range(MAX_ATTEMPTS):
|
|
check(handle(msg,force_error='temporary')=='retry','temporary failure not handled')
|
|
with store.connect() as conn:
|
|
j=fetch_job(conn,jid)
|
|
if attempt<MAX_ATTEMPTS-1:
|
|
delay=conn.execute('SELECT extract(epoch FROM next_at-now()) FROM jobs WHERE id=%s',(jid,)).fetchone()[0]
|
|
check(delay>0,'no backoff');delays.append(float(delay))
|
|
check(handle(msg)=='defer','retry ignored schedule')
|
|
conn.execute("UPDATE jobs SET next_at=now()-interval '1 second' WHERE id=%s",(jid,))
|
|
check(metadata(att)['error_code']=='retry_exhausted','retry was not bounded')
|
|
replay(store,jid);check(handle(msg)=='ready','retry exhausted replay failed');invariant(att,data)
|
|
rows.append(dict(scenario='bounded_retry_jitter',status='PASS',max_attempts=MAX_ATTEMPTS,delays_seconds=delays))
|
|
# COMMIT -> publish recovery, and a lost publish confirm duplicates safely.
|
|
att,data,jid,msg=upload()
|
|
with store.connect() as conn:check(conn.execute('SELECT published_at FROM outbox WHERE job_id=%s',(jid,)).fetchone()[0] is None,'outbox already published')
|
|
publish_once(r)
|
|
with store.connect() as conn:
|
|
conn.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(jid,))
|
|
publish_once(r)
|
|
check(handle(msg)=='ready' and handle(msg)=='noop','republish was unsafe');invariant(att,data)
|
|
rows.append(dict(scenario='outbox_republish',status='PASS',objects=4))
|
|
with store.connect() as conn:
|
|
accepted=conn.execute('SELECT count(*) FROM attachments').fetchone()[0]
|
|
result={'status':'PASS','track':2,'matrix_repeats':3,'rows':rows,'accepted_in_database':accepted,
|
|
'invariants':{'accepted_tasks_lost':0,'duplicate_variant_sets':0,'stale_version_overwrites':0},
|
|
'limitations':['matrix lease expiry accelerated with SQL','single host, worker processing is serial except intentional duplicate probe','no disk-loss or network partition proof']}
|
|
print(json.dumps(result,ensure_ascii=False,indent=2))
|
|
finally:r.close();store.pool.close();object_store.close()
|
|
|
|
|
|
if __name__=='__main__':main()
|