"""Run unchanged public resilience tests and record which Compose processes changed.""" import argparse import json import os from pathlib import Path import subprocess import sys import time ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client def docker(*args): return subprocess.run(['docker', *args], cwd=ROOT, capture_output=True, text=True, check=True, timeout=120).stdout def snapshot(): ids = docker('compose', 'ps', '--all', '-q').split() states = json.loads(docker('inspect', *ids)) return {state['Name'].lstrip('/'): { 'service': state['Config']['Labels']['com.docker.compose.service'], 'instance': state['Config']['Hostname'], 'startedAt': state['State']['StartedAt'], 'status': state['State']['Status'], 'restartCount': state['RestartCount'], } for state in states if state['Config']['Labels']['com.docker.compose.service'] in {'api', 'nginx', 'postgres', 'redis', 's3', 'reconciler'}} def public_test(mode): extension = 'cmd' if os.name == 'nt' else 'sh' hook = ROOT / 'scripts' / (('restart.' if mode == 'persistence' else 'stop-one.') + extension) result = subprocess.run([sys.executable, '-X', 'utf8', '-u', str(ROOT / 'tests/resilience.py'), '--mode', mode, '--action-script', str(hook), '--base-url', 'http://127.0.0.1:8080'], cwd=ROOT, capture_output=True, text=True, timeout=240) print(result.stdout, end='', flush=True) if result.returncode: raise RuntimeError(f'Public {mode} test failed: {result.stderr.strip() or result.stdout.strip()}') return result.stdout.strip() def warm_writes(): # Exercise writes and attachment I/O before measuring a steady-state API failure. client = Client('http://127.0.0.1:8080', timeout=10) alice, _, _, chat = client.bootstrap() timings = {} for index in range(30): result = client.message(alice, chat, f'Failover warm-up {index}') timings.setdefault(result['headers']['X-Instance-Id'], []).append(result['elapsed']) if len(timings) == 2 and min(map(len, timings.values())) >= 3: break assert len(timings) == 2 and min(map(len, timings.values())) >= 3, 'Warm-up did not reach both API write paths' result = {instance: {'writes': len(values), 'minSeconds': min(values), 'maxSeconds': max(values), 'uploadSeconds': [], 'downloadSeconds': []} for instance, values in timings.items()} image = (ROOT / 'tests/fixtures/sample.png').read_bytes() for index in range(30): uploaded = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=warmup.png', 201, actor=alice, raw=image, headers={'Content-Type': 'image/png'}, schema='Attachment') result[uploaded['headers']['X-Instance-Id']]['uploadSeconds'].append(uploaded['elapsed']) path = '/api/v1/attachments/' + uploaded['json']['id'] client.call('GET', path, actor=alice, schema='Attachment') downloaded = client.call('GET', path + '/content', actor=alice) assert downloaded['bytes'] == image, 'Warm-up attachment checksum mismatch' result[downloaded['headers']['X-Instance-Id']]['downloadSeconds'].append(downloaded['elapsed']) if all(entry['uploadSeconds'] and entry['downloadSeconds'] for entry in result.values()): break assert all(entry['uploadSeconds'] and entry['downloadSeconds'] for entry in result.values()), 'Attachment warm-up did not reach both replicas' return result def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--mode', choices=['all', 'failover'], default='all') args = parser.parse_args() report = {'status': 'failed', 'failover': []} report['writeWarmups'] = [] try: before = snapshot() assert sum(state['service'] == 'api' for state in before.values()) == 2, 'Expected two API replicas' assert all(state['status'] == 'running' for state in before.values()), 'Start the complete stack first' if args.mode == 'all': output = public_test('persistence') after = snapshot() assert before.keys() == after.keys(), 'Restart hook recreated runtime containers' assert all(state['startedAt'] != before[name]['startedAt'] and state['status'] == 'running' for name, state in after.items()), 'Hook did not restart every runtime service' report['persistence'] = {'before': before, 'after': after, 'output': output} else: previous = json.loads((ROOT / 'evidence/hooks.json').read_text(encoding='utf-8')) assert previous.get('persistence'), 'Run the full suite first to record persistence evidence' report['persistence'] = previous['persistence'] report['persistenceEvidenceReused'] = True after = before expected = {state['instance'] for state in after.values() if state['service'] == 'api'} stopped_instances = set() # The public test selects the replica that served its POST, so repeat independent # successful scenarios until each replica has been targeted. No failing run is retried. for _ in range(6): if stopped_instances == expected: break before = snapshot() try: report['writeWarmups'].append(warm_writes()) output = public_test('failover') after = snapshot() stopped = [state for state in after.values() if state['service'] == 'api' and state['status'] == 'exited'] assert len(stopped) == 1, 'Stop-one hook did not leave exactly one API stopped' assert all(state['startedAt'] == before[name]['startedAt'] and state['status'] == 'running' for name, state in after.items() if state['service'] != 'api'), 'Failover touched non-API services' target = stopped[0]['instance'] stopped_instances.add(target) report['failover'].append({'stoppedInstance': target, 'before': before, 'after': after, 'output': output}) finally: docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--no-build', '--wait', '--wait-timeout', '90', 'api') time.sleep(5) # Allow the Nginx readiness poll to observe the restored API. assert stopped_instances == expected, 'Public scenarios did not cover both API replicas' report['status'] = 'passed' print('PASS: public persistence/failover hooks; all stores restarted; each API stopped separately; replicas restored') except (AssertionError, RuntimeError, OSError, subprocess.SubprocessError) as error: report['error'] = str(error) if isinstance(error, (AssertionError, RuntimeError)) else type(error).__name__ raise finally: (ROOT / 'evidence/hooks.json').write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') if __name__ == '__main__': try: main() except (AssertionError, RuntimeError, OSError, subprocess.SubprocessError) as error: print('FAIL: ' + (str(error) if isinstance(error, (AssertionError, RuntimeError)) else type(error).__name__), file=sys.stderr) sys.exit(1)