forked from mtusi2026/mtusi2026lab4
57 lines
2.8 KiB
Python
57 lines
2.8 KiB
Python
"""Stop the actual uploader replica, then download through the survivor via LB."""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from client import Client,Failure,arguments,check,run
|
|
from resilience import history
|
|
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def main():
|
|
args=arguments(__doc__)
|
|
c=Client(args.base_url,timeout=15)
|
|
alice,bob,eve,chat=c.bootstrap()
|
|
data=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
response=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=failover.png',201,
|
|
actor=alice,raw=data,headers={'Content-Type':'image/png'},schema='Attachment')
|
|
att=response['json'];uploader=response['headers']['X-Instance-Id']
|
|
confirmed=c.message(alice,chat,'uploader failure',attachments=[att['id']])['json']
|
|
observed={c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id'] for _ in range(20)}
|
|
check(len(observed)>=2,'need two real processes')
|
|
proc=subprocess.Popen([str(ROOT/'scripts/stop-one.sh')],env=dict(os.environ,TARGET_INSTANCE_ID=uploader),stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
started=time.monotonic();errors=0;recovered=None;served=None
|
|
c.timeout=2
|
|
key=uuid.uuid4().hex
|
|
while time.monotonic()-started<10:
|
|
try:
|
|
result=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
|
|
check(result['bytes']==data,'original bytes changed')
|
|
served=result['headers']['X-Instance-Id']
|
|
c.message(alice,chat,'write during file failover',key=key)
|
|
if proc.poll()==0 and served!=uploader:
|
|
recovered=time.monotonic()-started
|
|
break
|
|
except Failure:
|
|
errors+=1
|
|
time.sleep(.1)
|
|
check(proc.wait(timeout=5)==0,'uploader hook failed')
|
|
check(recovered is not None and recovered<=10,'download/write did not recover within 10 seconds')
|
|
check(confirmed in history(c,bob,chat),'confirmed attachment message lost')
|
|
snapshot=json.loads(subprocess.check_output([sys.executable,str(ROOT/'scripts/container_snapshot.py')],text=True))
|
|
check(any(row['hostname']==uploader and row['status']=='exited' for row in snapshot),'uploader container was not actually stopped')
|
|
report={'status':'PASS','uploader_instance':uploader,'download_instance':served,
|
|
'observed_instances_before':sorted(observed),'failed_attempts':errors,
|
|
'first_download_write_recovery_seconds':round(recovered,3),
|
|
'sha256':hashlib.sha256(data).hexdigest(),'same_bytes':True,'containers_after_stop':snapshot}
|
|
(ROOT/'evidence/attachment-failover.json').write_text(json.dumps(report,indent=2)+'\n')
|
|
print(json.dumps({k:v for k,v in report.items() if k!='containers_after_stop'},indent=2))
|
|
|
|
if __name__=='__main__':
|
|
run(main)
|