Files
mtusi2026lab5/scripts/lab4/failover.py
T

96 lines
4.2 KiB
Python

"""Stop each actual uploader replica and verify its bytes through the survivor."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
def instances():
text = subprocess.check_output(['sh', 'scripts/api-instances.sh'], cwd=ROOT,
encoding='utf-8')
result = []
for line in text.splitlines():
container, replica, state, alias, image = line.split()
result.append({'container': container, 'replica': replica,
'state': state, 'alias': alias, 'image': image})
return result
def restore(c):
subprocess.run(['docker', 'compose', 'start', 'api'], cwd=ROOT,
check=True, stdout=subprocess.DEVNULL)
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
observed = {c.call('GET', '/health/live')['headers']['X-Instance-Id']
for _ in range(6)}
if len(observed) >= 2:
return
time.sleep(0.3)
raise AssertionError('two replicas did not return after restore')
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--base-url', default='http://localhost:8080')
p.add_argument('--output', type=Path, default=ROOT / 'evidence/failover-uploaders.json')
args = p.parse_args()
c = Client(args.base_url, timeout=10)
actor, reader, _, chat = c.bootstrap()
data = (ROOT / 'tests/fixtures/sample.png').read_bytes()
wanted = instances()
check(len(wanted) == 2 and all(x['state'] == 'running' for x in wanted),
'this experiment requires two running API replicas')
report = {'scenario': 'stop actual upload-serving replica; download via survivor',
'cases': [], 'ok': False}
try:
for target in wanted:
upload = None
for _ in range(10):
attempt = c.call('POST', '/api/v1/chats/' + chat['id'] +
'/attachments?filename=sample.png', 201,
actor=actor, raw=data,
headers={'Content-Type': 'image/png'}, schema='Attachment')
if attempt['headers']['X-Instance-Id'] == target['alias']:
upload = attempt
break
check(upload is not None, 'target replica did not serve an upload')
before = instances()
env = dict(os.environ, TARGET_INSTANCE_ID=target['alias'])
started = time.monotonic()
subprocess.run(['sh', 'scripts/stop-one.sh'], cwd=ROOT, env=env,
check=True, stdout=subprocess.DEVNULL, timeout=15)
stop_seconds = time.monotonic() - started
downloaded = c.call('GET', '/api/v1/attachments/' +
upload['json']['id'] + '/content', actor=reader)
survivor = downloaded['headers']['X-Instance-Id']
check(survivor != target['alias'], 'stopped uploader still served download')
check(downloaded['bytes'] == data, 'bytes changed after uploader stop')
after = instances()
check(any(x['container'] == target['container'] and x['state'] != 'running'
for x in after), 'uploader was not left stopped')
report['cases'].append({'uploader': target, 'download_instance': survivor,
'stop_seconds': round(stop_seconds, 3),
'sha256': hashlib.sha256(downloaded['bytes']).hexdigest(),
'same_bytes': True, 'before': before, 'after': after})
restore(c)
report['ok'] = True
finally:
# This script only restores the API containers that it deliberately stopped.
restore(c)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
print(json.dumps({'ok': report['ok'], 'cases': len(report['cases']),
'output': str(args.output)}))
if __name__ == '__main__':
main()