Files
mtusi2026lab6/student-tests/retry_backpressure.py
T

39 lines
3.0 KiB
Python

#!/usr/bin/env python3
"""Controlled transient failures, finite retries, admission bound, hard API kill."""
import json,time,uuid
from pipeline import ROOT,C,dc,sql,check,meta,wait_status,content,upload,start_worker
def main():
a,b,e,chat=C.bootstrap();result={}
dc('stop','worker','dispatcher');att,data=upload(a,chat)
dc('kill','-s','SIGKILL','api');dc('restart','broker');dc('up','-d','--no-deps','--wait','api','dispatcher','worker')
ready,recovery=wait_status(a,att['id'],'ready');check(content(a,att['id'])==data,'SIGKILL lost accepted original');result['sigkill_after_commit_before_publish']={'ready':True,'seconds':recovery}
# Hold S3 down while retry worker runs. Read metadata via SQL because S3 is
# intentionally a readiness dependency; no fake app success during this outage.
dc('stop','worker');bad,data=upload(a,chat);dc('stop','s3');start_worker();start=time.monotonic()
try:
deadline=time.monotonic()+130
while time.monotonic()<deadline:
status=sql("SELECT status FROM attachments WHERE id='"+bad['id']+"'")
if status=='failed':break
time.sleep(.5)
else:raise AssertionError('retries did not terminate')
attempts=int(sql("SELECT attempt FROM pipeline_jobs WHERE attachment_id='"+bad['id']+"'"));check(attempts==5,'wrong retry budget')
code=sql("SELECT error_code FROM attachments WHERE id='"+bad['id']+"'");check(code=='retry_exhausted','wrong terminal code')
events=json.loads(sql("SELECT json_agg(x) FROM (SELECT event,attempt,created_at FROM pipeline_audit WHERE job_id=(SELECT job_id FROM pipeline_jobs WHERE attachment_id='"+bad['id']+"') ORDER BY id) x"))
result['retry_exhaustion']={'attempts':attempts,'error_code':code,'seconds':round(time.monotonic()-start,3),'audit':events}
finally:dc('up','-d','--no-deps','--wait','s3');time.sleep(2)
dc('run','--rm','--no-deps','worker','replay',bad['id']);wait_status(a,bad['id'],'ready');check(content(a,bad['id'])==data,'replay after transient failure lost source')
# Fill the durable admission ledger with synthetic test-only entries in this
# test's own chat; delete them transactionally at the end (no API/user data).
dc('stop','worker','dispatcher');testid=uuid.uuid4().hex
try:
sql("INSERT INTO pipeline_jobs(job_id,attachment_id,pipeline_version) SELECT gen_random_uuid(),'"+att['id']+"',n FROM generate_series(100,1099)n")
C.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=backpressure.png',429,actor=a,raw=data,headers={'Content-Type':'image/png'})
check(sql("SELECT count(*) FROM pipeline_jobs WHERE pipeline_version BETWEEN 100 AND 1099 AND attachment_id='"+att['id']+"'")=='1000','fixture mutated')
result['admission_bound']={'unfinished_cap':1000,'response':429,'confirmed_work_retained':True}
finally:
sql("DELETE FROM pipeline_jobs WHERE pipeline_version BETWEEN 100 AND 1099 AND attachment_id='"+att['id']+"'");dc('up','-d','--no-deps','worker','dispatcher')
(ROOT/'evidence/retry-backpressure.json').write_text(json.dumps(result,indent=2)+'\n');print('PASS: SIGKILL, finite durable retry/backoff/replay, atomic admission limit')
if __name__=='__main__':main()