"""Host orchestration: stopped worker, API kill, broker restart, real lease.""" import json import os from pathlib import Path import subprocess import sys import time sys.path.insert(0,str(Path(__file__).resolve().parent)) from client import Client,check ROOT=Path(__file__).resolve().parents[1] COMPOSE=[str(ROOT/'scripts/compose.sh')] def compose(*args,allow_failure=False): p=subprocess.run(COMPOSE+list(args),cwd=ROOT,text=True,capture_output=True) if p.returncode and not allow_failure: raise AssertionError('Compose action failed: '+' '.join(args[:3])+': '+p.stderr[-500:]) return p def probe(aid): return json.loads(compose('run','--rm','--no-deps','ops','python','scripts/query_pipeline.py',aid).stdout)[-1] def wait_api(c): deadline=time.monotonic()+60 while time.monotonic()=length_before>0,'broker lost durable messages') after=snapshot(['api','broker']) check(all(after[name]['started_at']!=row['started_at'] and after[name]['volumes']==row['volumes'] for name,row in before.items()),'restart did not preserve volumes') start=time.monotonic();compose('start','worker');c.wait_ready(bob,att) check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'accepted original lost') rows.append(dict(scenario='stopped_worker_api_kill_outbox_broker_restart',status='PASS',attachment_id=att['id'], stream_length_before=length_before,stream_length_after=length_after,ready_ms=round((time.monotonic()-start)*1000,3), containers_before=before,containers_after=after)) compose('stop','worker','publisher') att,data=c.attachment(alice,chat);jid=probe(att['id'])['job_id'] crash=compose('run','--rm','--no-deps','ops','python','tests/drive_job.py',jid,'--crash','after_effect',allow_failure=True) check(crash.returncode==86,'worker failpoint did not crash') check(probe(att['id'])['state']=='processing','processing lease missing') start=time.monotonic();compose('start','publisher','worker');c.wait_ready(bob,att) elapsed=time.monotonic()-start check(elapsed>=20,'real lease expiry not exercised') check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'lease recovery corrupted original') rows.append(dict(scenario='crash_real_lease_expiry',status='PASS',attachment_id=att['id'],ready_ms=round(elapsed*1000,3), job=probe(att['id']),lease_clock='real')) print(json.dumps({'status':'PASS','rows':rows},ensure_ascii=False,indent=2)) finally:compose('start','api','broker','publisher','worker') if __name__=='__main__':main()