"""Stop each actual uploader replica and verify its bytes through the survivor.""" import argparse import hashlib import json import os from pathlib import Path import subprocess import sys import time ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, check def instances(): text = subprocess.check_output(['sh', 'scripts/api-instances.sh'], cwd=ROOT, encoding='utf-8') result = [] for line in text.splitlines(): container, replica, state, alias, image = line.split() result.append({'container': container, 'replica': replica, 'state': state, 'alias': alias, 'image': image}) return result def restore(c): subprocess.run(['docker', 'compose', 'start', 'api'], cwd=ROOT, check=True, stdout=subprocess.DEVNULL) deadline = time.monotonic() + 30 while time.monotonic() < deadline: observed = {c.call('GET', '/health/live')['headers']['X-Instance-Id'] for _ in range(6)} if len(observed) >= 2: return time.sleep(0.3) raise AssertionError('two replicas did not return after restore') def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('--base-url', default='http://localhost:8080') p.add_argument('--output', type=Path, default=ROOT / 'evidence/failover-uploaders.json') args = p.parse_args() c = Client(args.base_url, timeout=10) actor, reader, _, chat = c.bootstrap() data = (ROOT / 'tests/fixtures/sample.png').read_bytes() wanted = instances() check(len(wanted) == 2 and all(x['state'] == 'running' for x in wanted), 'this experiment requires two running API replicas') report = {'scenario': 'stop actual upload-serving replica; download via survivor', 'cases': [], 'ok': False} try: for target in wanted: upload = None for _ in range(10): attempt = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', 201, actor=actor, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment') if attempt['headers']['X-Instance-Id'] == target['alias']: upload = attempt break check(upload is not None, 'target replica did not serve an upload') before = instances() env = dict(os.environ, TARGET_INSTANCE_ID=target['alias']) started = time.monotonic() subprocess.run(['sh', 'scripts/stop-one.sh'], cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL, timeout=15) stop_seconds = time.monotonic() - started downloaded = c.call('GET', '/api/v1/attachments/' + upload['json']['id'] + '/content', actor=reader) survivor = downloaded['headers']['X-Instance-Id'] check(survivor != target['alias'], 'stopped uploader still served download') check(downloaded['bytes'] == data, 'bytes changed after uploader stop') after = instances() check(any(x['container'] == target['container'] and x['state'] != 'running' for x in after), 'uploader was not left stopped') report['cases'].append({'uploader': target, 'download_instance': survivor, 'stop_seconds': round(stop_seconds, 3), 'sha256': hashlib.sha256(downloaded['bytes']).hexdigest(), 'same_bytes': True, 'before': before, 'after': after}) restore(c) report['ok'] = True finally: # This script only restores the API containers that it deliberately stopped. restore(c) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') print(json.dumps({'ok': report['ok'], 'cases': len(report['cases']), 'output': str(args.output)})) if __name__ == '__main__': main()