53 lines
2.7 KiB
Python
53 lines
2.7 KiB
Python
"""Prove a real retry; start only the worker container, not its Compose dependencies."""
|
|
from pathlib import Path
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0,str(ROOT/'scripts'))
|
|
from observe_experiment import Experiment,command,save,query
|
|
def raw_start(service):
|
|
ids=command(['ps','-a','-q',service]).split()
|
|
assert len(ids)==1,'Expected one physical '+service
|
|
subprocess.run(['docker','start',ids[0]],check=True,capture_output=True,timeout=20)
|
|
def job_state(attachment_id):
|
|
sql="SELECT json_build_object('status',j.status,'attempts',j.attempts,'retry_events',(SELECT count(*) FROM job_events e WHERE e.job_id=j.id AND e.event='retry_scheduled')) FROM jobs j WHERE attachment_id='"+attachment_id+"'::uuid"
|
|
raw=command(['exec','-T','db','psql','-U','messenger','-d','messenger','-tAc',sql])
|
|
return json.loads(raw.strip())
|
|
def main():
|
|
exp=Experiment(1);exp.phase='verified_retry'
|
|
before=json.loads(subprocess.check_output(['docker','inspect',*command(['ps','-a','-q','worker','s3']).split()],text=True))
|
|
safe_before=[{'id':r['Id'],'name':r['Name'],'running':r['State']['Running']} for r in before]
|
|
state='FAILED';snapshots=[];upload=None
|
|
try:
|
|
command(['stop','worker']);upload=exp.upload()
|
|
command(['stop','s3'])
|
|
raw_start('worker')
|
|
deadline=time.monotonic()+20
|
|
while time.monotonic()<deadline:
|
|
row=job_state(upload['id']);snapshots.append({'time':time.time(),**row})
|
|
if row['retry_events']>=1 and row['attempts']>=1:break
|
|
time.sleep(.25)
|
|
else:raise AssertionError('No durable retry observed while S3 stopped')
|
|
s3_id=command(['ps','-a','-q','s3']).strip()
|
|
assert subprocess.check_output(['docker','inspect','--format','{{.State.Running}}',s3_id],text=True).strip()=='false'
|
|
raw_start('s3')
|
|
deadline=time.monotonic()+60
|
|
while time.monotonic()<deadline:
|
|
try:exp.client.call('GET','/health/ready',schema='Health');break
|
|
except Exception:time.sleep(.5)
|
|
else:raise AssertionError('API did not recover after S3 start')
|
|
exp.ready(upload)
|
|
completed=job_state(upload['id'])
|
|
assert completed['attempts']>=2 and completed['retry_events']>=1
|
|
time.sleep(15)
|
|
exp.observations('signals_retry.json')
|
|
state='PASS'
|
|
finally:
|
|
raw_start('s3');raw_start('worker')
|
|
save('retry_trace_result.json',{'status':state,'before':safe_before,'snapshots':snapshots,'upload':upload,
|
|
'final_job':job_state(upload['id']) if upload else None})
|
|
print('PASS: S3 truly stopped; durable retry event; same upload ready after >=2 attempts')
|
|
if __name__=='__main__':main()
|