#!/usr/bin/env python3 """Run from tests/ in lab3. Uses the unchanged skeleton client and resilience helpers. Creates synthetic users/messages, stops only API containers, restores in finally. Metrics are a small closed-loop workload, not a capacity benchmark. """ import csv import json import subprocess import threading import time import uuid from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone from pathlib import Path from client import Client, check from resilience import history ROOT = Path(__file__).resolve().parents[1] OUT = ROOT / 'results' / ('level4-' + datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')) def docker(*args): p = subprocess.run(['docker', *args], cwd=ROOT, text=True, capture_output=True, timeout=210) check(p.returncode == 0, 'Docker command failed: ' + ' '.join(args)) return p.stdout.strip() def state(cid): return json.loads(docker('inspect', '--format', '{{json .State}}', cid)) def ready(cid): deadline = time.monotonic() + 180 while time.monotonic() < deadline: s = state(cid) if s.get('Running') and s.get('Health', {}).get('Status') == 'healthy': return time.sleep(1) raise RuntimeError('API did not become healthy: ' + cid[:12]) def observe(c, actor, expected): deadline = time.monotonic() + 15 seen = set() while time.monotonic() < deadline: try: r = c.call('GET', '/api/v1/users/me', actor=actor, schema='User') seen.add(r['headers']['X-Instance-Id']) if expected <= seen: return except Exception: pass time.sleep(.1) raise RuntimeError('Both actual API hostnames must be observed through LB') def main(): ids = docker('compose', 'ps', '--status', 'running', '-q', 'api').splitlines() check(len(ids) == 2, 'Start exactly two healthy API replicas first') names = {cid: docker('inspect', '--format', '{{.Config.Hostname}}', cid) for cid in ids} for cid in ids: ready(cid) OUT.mkdir(parents=True) summary = {'workload': '4 closed-loop workers, POST then GET, 50ms pause; HTTP timeout 2s', 'replicas': names, 'experiments': []} c = Client(timeout=2) alice, bob, _, chat = c.bootstrap() path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100' observe(c, alice, set(names.values())) # One key, eight requests released together; a fresh chat isolates this test. barrier = threading.Barrier(8) key = uuid.uuid4().hex def replay(_): local = Client(timeout=10) barrier.wait(timeout=15) r = local.message(alice, chat, 'parallel-' + key, key=key) return {'instance': r['headers']['X-Instance-Id'], 'message': r['json']} with ThreadPoolExecutor(max_workers=8) as pool: replies = list(pool.map(replay, range(8))) check(all(r['message'] == replies[0]['message'] for r in replies), 'Concurrent replies differ') check(len({r['instance'] for r in replies}) == 2, 'Concurrency did not cover both replicas; rerun') check(history(c, bob, chat) == [replies[0]['message']], 'Concurrent replay created duplicates') summary['concurrency'] = {'pass': True, 'responses': replies} print('PASS: parallel replay, two replicas, one message', flush=True) for cid in ids: observe(c, alice, set(names.values())) rows, confirmed = [], {} halt = threading.Event() lock = threading.Lock() start = time.monotonic() def worker(): local = Client(timeout=2) key = uuid.uuid4().hex while not halt.is_set(): # Preserve the key/body after any uncertain POST result. for method in ('POST', 'GET'): began = time.monotonic() row = {'method': method, 'start': began - start, 'ok': False, 'instance': '', 'error': ''} try: if method == 'POST': r = local.message(alice, chat, 'load-' + key, key=key) with lock: confirmed[r['json']['id']] = r['json'] key = uuid.uuid4().hex else: r = local.call('GET', path, actor=alice, schema='MessagePage') row.update(ok=True, instance=r['headers']['X-Instance-Id']) except Exception as exc: row['error'] = type(exc).__name__ row['end'] = time.monotonic() - start with lock: rows.append(row) halt.wait(.05) pool = ThreadPoolExecutor(max_workers=4) jobs = [pool.submit(worker) for _ in range(4)] try: time.sleep(5) impact = time.monotonic() - start docker('stop', '--timeout', '20', cid) stopped = time.monotonic() - start stopped_state = state(cid) check(not stopped_state['Running'], 'Target is still running') print('Stopped ' + names[cid] + '; measuring for 15 seconds', flush=True) time.sleep(15) finally: halt.set() pool.shutdown(wait=True) # Restore even if HTTP validation or measurement fails. docker('start', cid) ready(cid) for job in jobs: job.result() observe(c, alice, set(names.values())) stored = history(c, bob, chat) by_id = {m['id']: m for m in stored} check(all(by_id.get(k) == v for k, v in confirmed.items()), 'Confirmed message lost') texts = [m['text'] for m in stored] check(len(texts) == len(set(texts)), 'Uncertain retry created duplicate') end = max(r['end'] for r in rows) def metrics(lo, hi): selected = [r for r in rows if lo <= r['start'] < hi] good = sum(r['ok'] for r in selected) return {'seconds': round(hi-lo, 3), 'attempts': len(selected), 'errors': len(selected)-good, 'attempt_rps': round(len(selected)/(hi-lo), 2), 'success_rps': round(good/(hi-lo), 2)} recovery = {} for method in ('POST', 'GET'): hits = [r['end']-impact for r in rows if r['method'] == method and r['ok'] and r['start'] >= stopped and r['instance'] != names[cid]] recovery[method] = round(min(hits), 3) if hits else None result = {'target': names[cid], 'exit_code': stopped_state['ExitCode'], 'stop_started_seconds': impact, 'stop_finished_seconds': stopped, 'baseline': metrics(0, impact), 'during_and_after_stop': metrics(impact, end), 'first_success_after_stop_seconds_from_action_start': recovery, 'confirmed_messages_preserved': True} summary['experiments'].append(result) with (OUT / (names[cid] + '.csv')).open('w', newline='') as f: w = csv.DictWriter(f, fieldnames=list(rows[0])) w.writeheader() w.writerows(rows) (OUT / 'summary.json').write_text(json.dumps(summary, indent=2), encoding='utf-8') print(json.dumps(result, indent=2), flush=True) check(all(v is not None and v <= 10 for v in recovery.values()), 'Read/write recovery exceeded 10s') print('PASS: replica restored, session and data preserved', flush=True) print('Results:', OUT) print('This checks SIGTERM failover and concurrency, NOT rolling replacement.') if __name__ == '__main__': try: main() except Exception as exc: print('FAIL:', str(exc)) raise SystemExit(1)