Files

42 lines
3.2 KiB
Python

#!/usr/bin/env python3
"""Induce transient missing-stage errors, observe bounded DLQ, restore and explicitly replay."""
import hashlib, json, subprocess, time, uuid
from pathlib import Path
from client import Client, arguments, check, run
ROOT=Path(__file__).resolve().parents[1]
def compose(*args,output=False):
return subprocess.check_output(['docker','compose',*args],cwd=ROOT,text=True) if output else subprocess.run(['docker','compose',*args],cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
def main():
a=arguments('Bounded retry, DLQ, and explicit failed-job replay')
compose('stop','worker')
try:
c=Client(a.base_url,a.ca_file); alice,bob,_,chat=c.bootstrap(); raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
path='/api/v1/chats/'+chat['id']+'/attachments?filename=retry.png'
att=c.call('POST',path,202,actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')['json']
aid=str(uuid.UUID(att['id'])); chatid=str(uuid.UUID(chat['id']))
sql="UPDATE attachments SET staging_key='staging/%s/%s/missing' WHERE id='%s'::uuid"%(chatid,aid,aid)
compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',sql)
compose('up','-d','--force-recreate','worker')
started=time.monotonic(); deadline=started+120; state=None
while time.monotonic()<deadline:
state=c.call('GET','/api/v1/attachments/'+aid,actor=bob,schema='Attachment')['json']
if state['status']=='failed': break
time.sleep(.5)
check(state and state['status']=='failed' and state['error_code']=='retry_exhausted','job did not reach bounded retry_exhausted state')
dlq=int(compose('exec','-T','broker','redis-cli','XLEN','attachment-jobs-dlq',output=True).strip())
check(dlq>=1,'terminal failure was not recorded in DLQ')
restore="UPDATE attachments SET staging_key='staging/%s/%s' WHERE id='%s'::uuid"%(chatid,aid,aid)
compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',restore)
compose('run','--rm','--no-deps','worker','python','scripts/replay_failed.py',aid)
ready=c.wait_ready(bob,att,wait_seconds=90)
check(ready['status']=='ready' and ready['sha256']==hashlib.sha256(raw).hexdigest(),'explicit replay did not complete original attachment')
# A successful logical job cannot be force-run through the failed-only operator command.
refused=subprocess.run(['docker','compose','run','--rm','--no-deps','worker','python','scripts/replay_failed.py',aid],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
check(refused.returncode!=0,'replay command accepted a ready job')
result={'attachment_id':aid,'induced_fault':'staging key temporarily pointed to missing object','terminal_error_code':'retry_exhausted','attempt_limit':5,'dlq_length_after_failure':dlq,'explicit_replay':'ready','ready_sha256_matches':True,'replay_of_ready_job':'refused','elapsed_seconds':round(time.monotonic()-started,3)}
(ROOT/'evidence/retry-replay.json').write_text(json.dumps(result,indent=2)+'\n'); print(json.dumps(result,indent=2))
finally: compose('up','-d','--force-recreate','worker')
if __name__=='__main__': run(main)