Files
mtusi2026lab6/scripts/retry_trace.py
T
2026-10-30 21:38:40 +03:00

53 lines
2.7 KiB
Python

"""Prove a real retry; start only the worker container, not its Compose dependencies."""
from pathlib import Path
import json
import subprocess
import sys
import time
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'scripts'))
from observe_experiment import Experiment,command,save,query
def raw_start(service):
ids=command(['ps','-a','-q',service]).split()
assert len(ids)==1,'Expected one physical '+service
subprocess.run(['docker','start',ids[0]],check=True,capture_output=True,timeout=20)
def job_state(attachment_id):
sql="SELECT json_build_object('status',j.status,'attempts',j.attempts,'retry_events',(SELECT count(*) FROM job_events e WHERE e.job_id=j.id AND e.event='retry_scheduled')) FROM jobs j WHERE attachment_id='"+attachment_id+"'::uuid"
raw=command(['exec','-T','db','psql','-U','messenger','-d','messenger','-tAc',sql])
return json.loads(raw.strip())
def main():
exp=Experiment(1);exp.phase='verified_retry'
before=json.loads(subprocess.check_output(['docker','inspect',*command(['ps','-a','-q','worker','s3']).split()],text=True))
safe_before=[{'id':r['Id'],'name':r['Name'],'running':r['State']['Running']} for r in before]
state='FAILED';snapshots=[];upload=None
try:
command(['stop','worker']);upload=exp.upload()
command(['stop','s3'])
raw_start('worker')
deadline=time.monotonic()+20
while time.monotonic()<deadline:
row=job_state(upload['id']);snapshots.append({'time':time.time(),**row})
if row['retry_events']>=1 and row['attempts']>=1:break
time.sleep(.25)
else:raise AssertionError('No durable retry observed while S3 stopped')
s3_id=command(['ps','-a','-q','s3']).strip()
assert subprocess.check_output(['docker','inspect','--format','{{.State.Running}}',s3_id],text=True).strip()=='false'
raw_start('s3')
deadline=time.monotonic()+60
while time.monotonic()<deadline:
try:exp.client.call('GET','/health/ready',schema='Health');break
except Exception:time.sleep(.5)
else:raise AssertionError('API did not recover after S3 start')
exp.ready(upload)
completed=job_state(upload['id'])
assert completed['attempts']>=2 and completed['retry_events']>=1
time.sleep(15)
exp.observations('signals_retry.json')
state='PASS'
finally:
raw_start('s3');raw_start('worker')
save('retry_trace_result.json',{'status':state,'before':safe_before,'snapshots':snapshots,'upload':upload,
'final_job':job_state(upload['id']) if upload else None})
print('PASS: S3 truly stopped; durable retry event; same upload ready after >=2 attempts')
if __name__=='__main__':main()