#!/usr/bin/env python3 from concurrent.futures import ThreadPoolExecutor import json import subprocess import threading import time import uuid from pathlib import Path from client import Client, Failure, arguments, check, run ROOT = Path(__file__).resolve().parent.parent def sh(*cmd): return subprocess.run(cmd, cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip() def api_containers(): out = sh('docker', 'compose', '--env-file', '.env', 'ps', '-q', 'api') return [x[:12] for x in out.split()] def wait_healthy(count, timeout=90): deadline = time.monotonic() + timeout while time.monotonic() < deadline: out = sh('docker', 'compose', '--env-file', '.env', 'ps', 'api', '--format', '{{.Status}}') if out.count('(healthy)') >= count: return time.sleep(1) raise Failure('replicas did not become healthy') class Load: def __init__(self, base, ca, concurrency): c = Client(base, ca) self.alice, _, _, self.chat = c.bootstrap() self.base, self.ca, self.n = base, ca, concurrency self.events = [] self.lock = threading.Lock() self.stop = threading.Event() self.t0 = time.monotonic() def worker(self): c = Client(self.base, self.ca, timeout=3) while not self.stop.is_set(): key = uuid.uuid4().hex first_error = None while not self.stop.is_set(): try: r = c.message(self.alice, self.chat, 'k-' + key, key=key) t = time.monotonic() - self.t0 with self.lock: self.events.append((t, True, r['headers']['X-Instance-Id'], first_error)) break except Failure: t = time.monotonic() - self.t0 first_error = first_error or t with self.lock: self.events.append((t, False, None, None)) time.sleep(0.05) def start(self): self.pool = ThreadPoolExecutor(max_workers=self.n) self.futures = [self.pool.submit(self.worker) for _ in range(self.n)] def finish(self): self.stop.set() self.pool.shutdown(wait=True) def summary(self, begin, end): ev = [e for e in self.events if begin <= e[0] < end] ok = [e for e in ev if e[1]] bad = [e for e in ev if not e[1]] span = max(end - begin, 1e-9) return { 'window_seconds': round(span, 2), 'successful_requests': len(ok), 'failed_attempts': len(bad), 'successful_rps': round(len(ok) / span, 1), 'last_error_offset_seconds': round(max(e[0] for e in bad) - begin, 2) if bad else None, 'instances': sorted({e[2] for e in ok}), } def instances_between(self, begin, end=float('inf')): return sorted({e[2] for e in self.events if e[1] and begin <= e[0] < end}) def failover_each(args): ids = api_containers() check(len(ids) >= 2, 'need at least two replicas') load = Load(args.base_url, args.ca_file, args.concurrency) load.start() time.sleep(args.warmup) results = [] base = load.summary(2, args.warmup) for target in ids: before = time.monotonic() - load.t0 sh('docker', 'stop', '-t', '10', target) stopped = time.monotonic() - load.t0 time.sleep(args.hold) after_stop = time.monotonic() - load.t0 window = load.summary(before, after_stop) sh('docker', 'start', target) wait_healthy(len(ids)) restored = time.monotonic() - load.t0 time.sleep(args.warmup) results.append({ 'stopped_replica': target, 'during_failure': window, 'seen_stopped_replica_after_stop': target in load.instances_between(stopped + 0.5, after_stop), 'stop_command_seconds': round(stopped - before, 2), 'restore_seconds': round(restored - after_stop, 2), 'after_restore': load.summary(restored + 1, restored + args.warmup), }) load.finish() return {'baseline': base, 'replicas': results} def rolling(args): old = api_containers() check(len(old) >= 2, 'need at least two replicas') n = len(old) load = Load(args.base_url, args.ca_file, args.concurrency) load.start() time.sleep(args.warmup) begin = time.monotonic() - load.t0 sh('scripts/rolling-update.sh') end = time.monotonic() - load.t0 time.sleep(args.warmup) load.finish() return { 'old_replicas': old, 'new_replicas': api_containers(), 'update_window': load.summary(begin, end), 'old_replicas_served_after_update': [i for i in load.instances_between(end) if i in old], } def main(): def configure(p): p.add_argument('--mode', choices=['failover-each', 'rolling'], required=True) p.add_argument('--concurrency', type=int, default=8) p.add_argument('--warmup', type=float, default=8) p.add_argument('--hold', type=float, default=12) p.add_argument('--output') args = arguments('Failure and rolling update under load', configure) report = failover_each(args) if args.mode == 'failover-each' else rolling(args) text = json.dumps(report, indent=2) if args.output: Path(args.output).write_text(text + '\n') print(text) if __name__ == '__main__': run(main)