import argparse from concurrent.futures import ThreadPoolExecutor import json import os from pathlib import Path import subprocess import sys import time ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) sys.path.insert(0, str(ROOT / 'scripts')) from client import Client, Failure, check from operate import containers OUT = ROOT / 'evidence' def run(args, name): result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding='utf-8') (OUT / (name + '.txt')).write_text(result.stdout + result.stderr, encoding='utf-8') if result.returncode: raise RuntimeError(name + ': ' + result.stdout[-1000:] + result.stderr[-1000:]) print(name + ': PASS', flush=True) def restore(): subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--scale', 'api=2', '--wait', 'api'], cwd=ROOT, check=True, capture_output=True) time.sleep(4) def snapshot(): ids = subprocess.check_output(['docker', 'compose', 'ps', '-q'], cwd=ROOT, text=True).split() inspected = json.loads(subprocess.check_output(['docker', 'inspect', *ids], cwd=ROOT, text=True)) return [{'id': item['Id'][:12], 'service': item['Config']['Labels']['com.docker.compose.service'], 'hostname': item['Config']['Hostname'], 'started_at': item['State']['StartedAt'], 'status': item['State']['Status'], 'cpu_nanoseconds': item['HostConfig']['NanoCpus'], 'memory_bytes': item['HostConfig']['Memory'], 'pool_size': next((value.split('=', 1)[1] for value in item['Config']['Env'] if value.startswith('DB_POOL_SIZE=')), None), 'published_ports': item['HostConfig']['PortBindings']} for item in inspected] def matrix(): c = Client(timeout=2) alice, bob, eve, chat = c.bootstrap() confirmed = c.message(alice, chat, 'before matrix')['json'] rows = [] for item in containers(): restore() target = item['Config']['Hostname'] started = time.monotonic() errors, successes, reads, recovery, observed = 0, [], 0, None, set() with ThreadPoolExecutor(max_workers=1) as pool: action = pool.submit(subprocess.run, ['docker', 'kill', item['Id']], cwd=ROOT, check=True, capture_output=True) key = os.urandom(16).hex() while time.monotonic() - started < 12: try: result = c.message(alice, chat, 'matrix-' + key, key=key) successes.append(result['json']) observed.add(result['headers']['X-Instance-Id']) key = os.urandom(16).hex() page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=bob, schema='MessagePage')['json'] check(confirmed in page['items'], 'confirmed data missing during failure') reads += 1 if action.done() and recovery is None: recovery = time.monotonic() - started except Failure: errors += 1 time.sleep(0.1) action.result() state = json.loads(subprocess.check_output(['docker', 'inspect', '--format', '{{json .State}}', item['Id']], text=True)) check(not state['Running'], 'target still running') check(recovery is not None and recovery <= 10, 'recovery deadline exceeded') from resilience import history stored = history(c, bob, chat) check(all(message in stored for message in successes), 'lost confirmed write') texts = [message['text'] for message in stored] check(len(texts) == len(set(texts)), 'retry duplicated data') elapsed = time.monotonic() - started rows.append({'target': target, 'container': item['Id'][:12], 'state': state['Status'], 'successful_writes': len(successes), 'successful_reads': reads, 'failed_attempts': errors, 'elapsed_seconds': round(elapsed, 3), 'write_rps': round(len(successes)/elapsed, 3), 'read_rps': round(reads/elapsed, 3), 'recovery_seconds': round(recovery, 3), 'observed_instances': sorted(observed), 'lost_confirmations': 0}) print('Hard failure verified: ' + target, flush=True) restore() (OUT / 'failover-matrix.json').write_text(json.dumps(rows, indent=2), encoding='utf-8') def rolling(): ready_file = OUT / 'rolling-ready.json' if ready_file.exists(): ready_file.unlink() with ThreadPoolExecutor(max_workers=1) as pool: load = pool.submit(run, [sys.executable, 'scripts/load_active.py', '--duration', '45', '--concurrency', '16', '--output', 'evidence/rolling-load.json', '--ready-file', str(ready_file)], 'rolling-load') deadline = time.monotonic() + 90 while not ready_file.exists(): if load.done(): load.result() raise RuntimeError('Load ended before measurement window started') check(time.monotonic() < deadline, 'load setup exceeded 90 seconds') time.sleep(0.2) run([sys.executable, 'scripts/operate.py', 'rolling'], 'rolling-action') load.result() summary = json.loads((OUT / 'rolling-load.json').read_text(encoding='utf-8')) replacements = [json.loads(line) for line in (OUT / 'rolling-action.txt').read_text(encoding='utf-8').splitlines() if line.startswith('{')] check(len(replacements) == 2, 'rolling must replace exactly two processes') check(all(summary['measurement_started_epoch'] <= row['started_epoch'] <= row['finished_epoch'] <= summary['measurement_finished_epoch'] for row in replacements), 'replacement happened outside active load window') run([sys.executable, 'tests/smoke.py'], 'smoke-after-rolling') def main(): parser = argparse.ArgumentParser() parser.add_argument('--rolling-only', action='store_true') args = parser.parse_args() OUT.mkdir(exist_ok=True) if not args.rolling_only: run([sys.executable, 'scripts/check_contract.py'], 'contract') run([sys.executable, 'tests/smoke.py'], 'smoke') run([sys.executable, 'student-tests/acceptance.py'], 'concurrent') suffix = '.cmd' if os.name == 'nt' else '.sh' before = snapshot() run([sys.executable, 'tests/resilience.py', '--mode', 'persistence', '--action-script', 'scripts/restart' + suffix], 'persistence') after = snapshot() check(all(any(new['id'] == old['id'] and new['started_at'] != old['started_at'] for new in after) for old in before), 'restart hook did not restart all running services') (OUT / 'restart-processes.json').write_text(json.dumps({'before': before, 'after': after}, indent=2), encoding='utf-8') restore() run([sys.executable, 'tests/resilience.py', '--mode', 'failover', '--action-script', 'scripts/stop-one' + suffix], 'public-failover') restore() matrix() rolling() if __name__ == '__main__': main()