Files
mtusi2026lab4/tests/attachment_failover.py

57 lines
2.8 KiB
Python

"""Stop the actual uploader replica, then download through the survivor via LB."""
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
import uuid
from client import Client,Failure,arguments,check,run
from resilience import history
ROOT=Path(__file__).resolve().parents[1]
def main():
args=arguments(__doc__)
c=Client(args.base_url,timeout=15)
alice,bob,eve,chat=c.bootstrap()
data=(ROOT/'tests/fixtures/sample.png').read_bytes()
response=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=failover.png',201,
actor=alice,raw=data,headers={'Content-Type':'image/png'},schema='Attachment')
att=response['json'];uploader=response['headers']['X-Instance-Id']
confirmed=c.message(alice,chat,'uploader failure',attachments=[att['id']])['json']
observed={c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id'] for _ in range(20)}
check(len(observed)>=2,'need two real processes')
proc=subprocess.Popen([str(ROOT/'scripts/stop-one.sh')],env=dict(os.environ,TARGET_INSTANCE_ID=uploader),stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
started=time.monotonic();errors=0;recovered=None;served=None
c.timeout=2
key=uuid.uuid4().hex
while time.monotonic()-started<10:
try:
result=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
check(result['bytes']==data,'original bytes changed')
served=result['headers']['X-Instance-Id']
c.message(alice,chat,'write during file failover',key=key)
if proc.poll()==0 and served!=uploader:
recovered=time.monotonic()-started
break
except Failure:
errors+=1
time.sleep(.1)
check(proc.wait(timeout=5)==0,'uploader hook failed')
check(recovered is not None and recovered<=10,'download/write did not recover within 10 seconds')
check(confirmed in history(c,bob,chat),'confirmed attachment message lost')
snapshot=json.loads(subprocess.check_output([sys.executable,str(ROOT/'scripts/container_snapshot.py')],text=True))
check(any(row['hostname']==uploader and row['status']=='exited' for row in snapshot),'uploader container was not actually stopped')
report={'status':'PASS','uploader_instance':uploader,'download_instance':served,
'observed_instances_before':sorted(observed),'failed_attempts':errors,
'first_download_write_recovery_seconds':round(recovered,3),
'sha256':hashlib.sha256(data).hexdigest(),'same_bytes':True,'containers_after_stop':snapshot}
(ROOT/'evidence/attachment-failover.json').write_text(json.dumps(report,indent=2)+'\n')
print(json.dumps({k:v for k,v in report.items() if k!='containers_after_stop'},indent=2))
if __name__=='__main__':
run(main)