"""Stop the actual uploader replica, then download through the survivor via LB.""" import hashlib import json import os from pathlib import Path import subprocess import sys import time import uuid from client import Client,Failure,arguments,check,run from resilience import history ROOT=Path(__file__).resolve().parents[1] def main(): args=arguments(__doc__) c=Client(args.base_url,timeout=15) alice,bob,eve,chat=c.bootstrap() data=(ROOT/'tests/fixtures/sample.png').read_bytes() response=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=failover.png',201, actor=alice,raw=data,headers={'Content-Type':'image/png'},schema='Attachment') att=response['json'];uploader=response['headers']['X-Instance-Id'] confirmed=c.message(alice,chat,'uploader failure',attachments=[att['id']])['json'] observed={c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id'] for _ in range(20)} check(len(observed)>=2,'need two real processes') proc=subprocess.Popen([str(ROOT/'scripts/stop-one.sh')],env=dict(os.environ,TARGET_INSTANCE_ID=uploader),stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) started=time.monotonic();errors=0;recovered=None;served=None c.timeout=2 key=uuid.uuid4().hex while time.monotonic()-started<10: try: result=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob) check(result['bytes']==data,'original bytes changed') served=result['headers']['X-Instance-Id'] c.message(alice,chat,'write during file failover',key=key) if proc.poll()==0 and served!=uploader: recovered=time.monotonic()-started break except Failure: errors+=1 time.sleep(.1) check(proc.wait(timeout=5)==0,'uploader hook failed') check(recovered is not None and recovered<=10,'download/write did not recover within 10 seconds') check(confirmed in history(c,bob,chat),'confirmed attachment message lost') snapshot=json.loads(subprocess.check_output([sys.executable,str(ROOT/'scripts/container_snapshot.py')],text=True)) check(any(row['hostname']==uploader and row['status']=='exited' for row in snapshot),'uploader container was not actually stopped') report={'status':'PASS','uploader_instance':uploader,'download_instance':served, 'observed_instances_before':sorted(observed),'failed_attempts':errors, 'first_download_write_recovery_seconds':round(recovered,3), 'sha256':hashlib.sha256(data).hexdigest(),'same_bytes':True,'containers_after_stop':snapshot} (ROOT/'evidence/attachment-failover.json').write_text(json.dumps(report,indent=2)+'\n') print(json.dumps({k:v for k,v in report.items() if k!='containers_after_stop'},indent=2)) if __name__=='__main__': run(main)