39 lines
3.0 KiB
Python
39 lines
3.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Controlled transient failures, finite retries, admission bound, hard API kill."""
|
|
import json,time,uuid
|
|
from pipeline import ROOT,C,dc,sql,check,meta,wait_status,content,upload,start_worker
|
|
|
|
def main():
|
|
a,b,e,chat=C.bootstrap();result={}
|
|
dc('stop','worker','dispatcher');att,data=upload(a,chat)
|
|
dc('kill','-s','SIGKILL','api');dc('restart','broker');dc('up','-d','--no-deps','--wait','api','dispatcher','worker')
|
|
ready,recovery=wait_status(a,att['id'],'ready');check(content(a,att['id'])==data,'SIGKILL lost accepted original');result['sigkill_after_commit_before_publish']={'ready':True,'seconds':recovery}
|
|
# Hold S3 down while retry worker runs. Read metadata via SQL because S3 is
|
|
# intentionally a readiness dependency; no fake app success during this outage.
|
|
dc('stop','worker');bad,data=upload(a,chat);dc('stop','s3');start_worker();start=time.monotonic()
|
|
try:
|
|
deadline=time.monotonic()+130
|
|
while time.monotonic()<deadline:
|
|
status=sql("SELECT status FROM attachments WHERE id='"+bad['id']+"'")
|
|
if status=='failed':break
|
|
time.sleep(.5)
|
|
else:raise AssertionError('retries did not terminate')
|
|
attempts=int(sql("SELECT attempt FROM pipeline_jobs WHERE attachment_id='"+bad['id']+"'"));check(attempts==5,'wrong retry budget')
|
|
code=sql("SELECT error_code FROM attachments WHERE id='"+bad['id']+"'");check(code=='retry_exhausted','wrong terminal code')
|
|
events=json.loads(sql("SELECT json_agg(x) FROM (SELECT event,attempt,created_at FROM pipeline_audit WHERE job_id=(SELECT job_id FROM pipeline_jobs WHERE attachment_id='"+bad['id']+"') ORDER BY id) x"))
|
|
result['retry_exhaustion']={'attempts':attempts,'error_code':code,'seconds':round(time.monotonic()-start,3),'audit':events}
|
|
finally:dc('up','-d','--no-deps','--wait','s3');time.sleep(2)
|
|
dc('run','--rm','--no-deps','worker','replay',bad['id']);wait_status(a,bad['id'],'ready');check(content(a,bad['id'])==data,'replay after transient failure lost source')
|
|
# Fill the durable admission ledger with synthetic test-only entries in this
|
|
# test's own chat; delete them transactionally at the end (no API/user data).
|
|
dc('stop','worker','dispatcher');testid=uuid.uuid4().hex
|
|
try:
|
|
sql("INSERT INTO pipeline_jobs(job_id,attachment_id,pipeline_version) SELECT gen_random_uuid(),'"+att['id']+"',n FROM generate_series(100,1099)n")
|
|
C.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=backpressure.png',429,actor=a,raw=data,headers={'Content-Type':'image/png'})
|
|
check(sql("SELECT count(*) FROM pipeline_jobs WHERE pipeline_version BETWEEN 100 AND 1099 AND attachment_id='"+att['id']+"'")=='1000','fixture mutated')
|
|
result['admission_bound']={'unfinished_cap':1000,'response':429,'confirmed_work_retained':True}
|
|
finally:
|
|
sql("DELETE FROM pipeline_jobs WHERE pipeline_version BETWEEN 100 AND 1099 AND attachment_id='"+att['id']+"'");dc('up','-d','--no-deps','worker','dispatcher')
|
|
(ROOT/'evidence/retry-backpressure.json').write_text(json.dumps(result,indent=2)+'\n');print('PASS: SIGKILL, finite durable retry/backoff/replay, atomic admission limit')
|
|
if __name__=='__main__':main()
|