Files
mtusi2026lab5/tests/lifecycle.py
T

85 lines
4.4 KiB
Python

"""Host orchestration: stopped worker, API kill, broker restart, real lease."""
import json
import os
from pathlib import Path
import subprocess
import sys
import time
sys.path.insert(0,str(Path(__file__).resolve().parent))
from client import Client,check
ROOT=Path(__file__).resolve().parents[1]
COMPOSE=[str(ROOT/'scripts/compose.sh')]
def compose(*args,allow_failure=False):
p=subprocess.run(COMPOSE+list(args),cwd=ROOT,text=True,capture_output=True)
if p.returncode and not allow_failure:
raise AssertionError('Compose action failed: '+' '.join(args[:3])+': '+p.stderr[-500:])
return p
def probe(aid):
return json.loads(compose('run','--rm','--no-deps','ops','python','scripts/query_pipeline.py',aid).stdout)[-1]
def wait_api(c):
deadline=time.monotonic()+60
while time.monotonic()<deadline:
try:c.call('GET','/health/ready',schema='Health');return
except Exception:time.sleep(.3)
raise AssertionError('API recovery timeout')
def snapshot(services):
ids=compose('ps','-q',*services).stdout.split()
state=json.loads(subprocess.check_output(['docker','inspect',*ids],text=True))
return {x['Name'].lstrip('/'):{'id':x['Id'][:12],'started_at':x['State']['StartedAt'],
'volumes':sorted(m['Name'] for m in x['Mounts'] if m['Type']=='volume')} for x in state}
def main():
c=Client(os.environ.get('BASE_URL','http://localhost:8085'))
rows=[]
try:
compose('stop','worker','publisher')
alice,bob,eve,chat=c.bootstrap();att,data=c.attachment(alice,chat)
message=c.message(alice,chat,'',attachments=[att['id']])['json']
c.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=alice)
check(probe(att['id'])['published_at'] is None,'publisher unexpectedly active')
before=snapshot(['api','broker'])
compose('kill','-s','SIGKILL','api');compose('start','api');wait_api(c)
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
page=c.call('GET','/api/v1/chats/'+chat['id']+'/messages',actor=bob,schema='MessagePage')['json']
check(any(m['id']==message['id'] for m in page['items']),'queued message lost')
compose('start','publisher')
deadline=time.monotonic()+20
while probe(att['id'])['published_at'] is None:
check(time.monotonic()<deadline,'outbox not delivered');time.sleep(.2)
length_before=int(compose('exec','-T','broker','valkey-cli','XLEN','attachments.v1').stdout)
compose('restart','broker')
length_after=int(compose('exec','-T','broker','valkey-cli','XLEN','attachments.v1').stdout)
check(length_after>=length_before>0,'broker lost durable messages')
after=snapshot(['api','broker'])
check(all(after[name]['started_at']!=row['started_at'] and after[name]['volumes']==row['volumes'] for name,row in before.items()),'restart did not preserve volumes')
start=time.monotonic();compose('start','worker');c.wait_ready(bob,att)
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'accepted original lost')
rows.append(dict(scenario='stopped_worker_api_kill_outbox_broker_restart',status='PASS',attachment_id=att['id'],
stream_length_before=length_before,stream_length_after=length_after,ready_ms=round((time.monotonic()-start)*1000,3),
containers_before=before,containers_after=after))
compose('stop','worker','publisher')
att,data=c.attachment(alice,chat);jid=probe(att['id'])['job_id']
crash=compose('run','--rm','--no-deps','ops','python','tests/drive_job.py',jid,'--crash','after_effect',allow_failure=True)
check(crash.returncode==86,'worker failpoint did not crash')
check(probe(att['id'])['state']=='processing','processing lease missing')
start=time.monotonic();compose('start','publisher','worker');c.wait_ready(bob,att)
elapsed=time.monotonic()-start
check(elapsed>=20,'real lease expiry not exercised')
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'lease recovery corrupted original')
rows.append(dict(scenario='crash_real_lease_expiry',status='PASS',attachment_id=att['id'],ready_ms=round(elapsed*1000,3),
job=probe(att['id']),lease_clock='real'))
print(json.dumps({'status':'PASS','rows':rows},ensure_ascii=False,indent=2))
finally:compose('start','api','broker','publisher','worker')
if __name__=='__main__':main()