forked from mtusi2026/mtusi2026lab5
42 lines
3.2 KiB
Python
42 lines
3.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Induce transient missing-stage errors, observe bounded DLQ, restore and explicitly replay."""
|
|
import hashlib, json, subprocess, time, uuid
|
|
from pathlib import Path
|
|
from client import Client, arguments, check, run
|
|
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
def compose(*args,output=False):
|
|
return subprocess.check_output(['docker','compose',*args],cwd=ROOT,text=True) if output else subprocess.run(['docker','compose',*args],cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
def main():
|
|
a=arguments('Bounded retry, DLQ, and explicit failed-job replay')
|
|
compose('stop','worker')
|
|
try:
|
|
c=Client(a.base_url,a.ca_file); alice,bob,_,chat=c.bootstrap(); raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
path='/api/v1/chats/'+chat['id']+'/attachments?filename=retry.png'
|
|
att=c.call('POST',path,202,actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')['json']
|
|
aid=str(uuid.UUID(att['id'])); chatid=str(uuid.UUID(chat['id']))
|
|
sql="UPDATE attachments SET staging_key='staging/%s/%s/missing' WHERE id='%s'::uuid"%(chatid,aid,aid)
|
|
compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',sql)
|
|
compose('up','-d','--force-recreate','worker')
|
|
started=time.monotonic(); deadline=started+120; state=None
|
|
while time.monotonic()<deadline:
|
|
state=c.call('GET','/api/v1/attachments/'+aid,actor=bob,schema='Attachment')['json']
|
|
if state['status']=='failed': break
|
|
time.sleep(.5)
|
|
check(state and state['status']=='failed' and state['error_code']=='retry_exhausted','job did not reach bounded retry_exhausted state')
|
|
dlq=int(compose('exec','-T','broker','redis-cli','XLEN','attachment-jobs-dlq',output=True).strip())
|
|
check(dlq>=1,'terminal failure was not recorded in DLQ')
|
|
restore="UPDATE attachments SET staging_key='staging/%s/%s' WHERE id='%s'::uuid"%(chatid,aid,aid)
|
|
compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',restore)
|
|
compose('run','--rm','--no-deps','worker','python','scripts/replay_failed.py',aid)
|
|
ready=c.wait_ready(bob,att,wait_seconds=90)
|
|
check(ready['status']=='ready' and ready['sha256']==hashlib.sha256(raw).hexdigest(),'explicit replay did not complete original attachment')
|
|
# A successful logical job cannot be force-run through the failed-only operator command.
|
|
refused=subprocess.run(['docker','compose','run','--rm','--no-deps','worker','python','scripts/replay_failed.py',aid],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
check(refused.returncode!=0,'replay command accepted a ready job')
|
|
result={'attachment_id':aid,'induced_fault':'staging key temporarily pointed to missing object','terminal_error_code':'retry_exhausted','attempt_limit':5,'dlq_length_after_failure':dlq,'explicit_replay':'ready','ready_sha256_matches':True,'replay_of_ready_job':'refused','elapsed_seconds':round(time.monotonic()-started,3)}
|
|
(ROOT/'evidence/retry-replay.json').write_text(json.dumps(result,indent=2)+'\n'); print(json.dumps(result,indent=2))
|
|
finally: compose('up','-d','--force-recreate','worker')
|
|
|
|
if __name__=='__main__': run(main)
|