#!/usr/bin/env python3 """Closed-loop load through the LB while API replicas are stopped one by one (or an action script runs). failover: for every running api container in turn: baseline -> docker stop (SIGTERM, drain) -> observe -> docker start the same container -> wait until it serves again -> settle. Restores before the next one. action: baseline -> run --action script (e.g. rolling-update) under load -> observe. Workers alternate POST message (Idempotency-Key; an uncertain result is retried with the SAME key and body) and GET history page. Per-second successes/errors and serving X-Instance-Id / X-App-Version are recorded. At the end the whole chat history is read back: every confirmed message must be present exactly once. Session tokens stay in memory and are never printed. """ import argparse import json import subprocess import sys import threading import time import uuid from collections import Counter, defaultdict from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure, check, run # noqa: E402 from resilience import history # noqa: E402 PROJECT = 'messenger-lab3' def docker(*args, timeout=60): return subprocess.run(['docker', *args], capture_output=True, text=True, timeout=timeout, check=True).stdout.strip() def api_containers(): """(name, short id) of running api containers of this Compose project, sorted by name.""" out = docker('ps', '--filter', 'label=com.docker.compose.project=' + PROJECT, '--filter', 'label=com.docker.compose.service=api', '--format', '{{.Names}} {{.ID}}') return sorted(tuple(line.split()) for line in out.splitlines() if line.strip()) class Load: def __init__(self, base_url, concurrency): self.base_url = base_url self.concurrency = concurrency self.events = [] # (t, op, ok, instance, version, latency) self.confirmed = {} # message id -> text self.lock = threading.Lock() self.stop = threading.Event() self.t0 = None def now(self): return time.monotonic() - self.t0 def record(self, op, ok, response=None, latency=0.0): headers = response['headers'] if response else {} event = (self.now(), op, ok, headers.get('X-Instance-Id') if response else None, headers.get('X-App-Version') if response else None, latency) with self.lock: self.events.append(event) def worker(self, worker_id, alice, chat): c = Client(self.base_url, timeout=5) path = '/api/v1/chats/' + chat['id'] + '/messages?limit=20' index = 0 key = text = None while not self.stop.is_set(): if key is None: key = uuid.uuid4().hex text = 'series-%d-%d-%s' % (worker_id, index, key[:8]) start = time.monotonic() try: response = c.message(alice, chat, text, key=key) self.record('write', True, response, time.monotonic() - start) with self.lock: self.confirmed[response['json']['id']] = text key = None index += 1 except Failure: self.record('write', False, latency=time.monotonic() - start) time.sleep(0.05) continue # same key and text: the first attempt may have committed start = time.monotonic() try: response = c.call('GET', path, actor=alice, schema='MessagePage') self.record('read', True, response, time.monotonic() - start) except Failure: self.record('read', False, latency=time.monotonic() - start) time.sleep(0.05) def start(self, alice, chat): self.t0 = time.monotonic() self.threads = [threading.Thread(target=self.worker, args=(i, alice, chat), daemon=True) for i in range(self.concurrency)] for t in self.threads: t.start() def finish(self): self.stop.set() for t in self.threads: t.join(timeout=10) def window(self, start, end): return [e for e in self.events if start <= e[0] < end] def summarize(events, start, end): ok = [e for e in events if e[2]] duration = max(end - start, 1e-9) latencies = sorted(e[5] for e in ok) p95 = round(1000 * latencies[max(0, int(0.95 * len(latencies)) - 1)], 1) if latencies else None return { 'seconds': round(duration, 2), 'rps_ok': round(len(ok) / duration, 1), 'writes_ok': sum(1 for e in ok if e[1] == 'write'), 'reads_ok': sum(1 for e in ok if e[1] == 'read'), 'errors': sum(1 for e in events if not e[2]), 'p95_ms': p95, 'instances': dict(Counter(e[3] for e in ok)), } def timeline(load, end): buckets = defaultdict(lambda: {'ok': 0, 'err': 0, 'instances': Counter(), 'versions': Counter()}) for t, _, ok, instance, version, _ in load.events: if t >= end: continue b = buckets[int(t)] if ok: b['ok'] += 1 b['instances'][instance] += 1 b['versions'][version] += 1 else: b['err'] += 1 return [{'t': s, 'ok': buckets[s]['ok'], 'err': buckets[s]['err'], 'instances': dict(buckets[s]['instances']), 'versions': dict(buckets[s]['versions'])} for s in range(int(end) + 1)] def max_gap(events, start, end): """Longest interval without any successful response inside [start, end).""" times = [start] + sorted(e[0] for e in events if e[2] and start <= e[0] < end) + [end] return round(max(b - a for a, b in zip(times, times[1:])), 3) def wait_serving(load, instance, since, limit=30): deadline = time.monotonic() + limit while time.monotonic() < deadline: with load.lock: hit = next((e[0] for e in reversed(load.events) if e[2] and e[3] == instance and e[0] >= since), None) if hit is not None: return hit time.sleep(0.2) return None def failover_run(load, name, cid, baseline, observe, settle, stop_timeout, kill): time.sleep(baseline) t_stop = load.now() if kill: docker('kill', cid) else: docker('stop', '--time', str(stop_timeout), cid, timeout=stop_timeout + 30) t_stopped = load.now() state = docker('inspect', '--format', '{{.State.Status}} exit={{.State.ExitCode}}', cid) time.sleep(observe) t_start = load.now() docker('start', cid) rejoined = wait_serving(load, cid, t_start) time.sleep(settle) t_end = load.now() with load.lock: events = list(load.events) base = summarize([e for e in events if t_stop - baseline <= e[0] < t_stop], t_stop - baseline, t_stop) outage = [e for e in events if t_stop <= e[0] < t_start] last_target = max((e[0] for e in outage if e[2] and e[3] == cid), default=None) first_ok_after_stop = min((e[0] for e in outage if e[2] and e[0] >= t_stopped), default=None) # Recovery: first full second after the container stopped whose throughput is >= 90% of baseline. recovered = None for s in range(int(t_stopped), int(t_start)): n = sum(1 for e in outage if e[2] and s <= e[0] < s + 1) if n >= 0.9 * base['rps_ok']: recovered = s + 1 - t_stopped break return { 'replica': name, 'instance_id': cid, 'state_after_stop': state, 'stop_command_seconds': round(t_stopped - t_stop, 2), 'last_served_by_target_after_sigterm_s': round(last_target - t_stop, 2) if last_target else None, 'first_success_after_container_exit_s': round(first_ok_after_stop - t_stopped, 3) if first_ok_after_stop else None, 'rps_back_to_90pct_after_exit_s': round(recovered, 2) if recovered is not None else None, 'max_gap_without_success_s': max_gap(events, t_stop, t_start), 'rejoin_after_docker_start_s': round(rejoined - t_start, 2) if rejoined else None, 'baseline': base, 'during_stop_and_outage': summarize(outage, t_stop, t_start), 'after_restore': summarize([e for e in events if t_start <= e[0] < t_end], t_start, t_end), 'marks': {'sigterm': round(t_stop, 2), 'exited': round(t_stopped, 2), 'docker_start': round(t_start, 2)}, } def action_run(load, action, baseline, observe): time.sleep(baseline) t_action = load.now() proc = subprocess.run([str(Path(action).resolve())], capture_output=True, text=True, timeout=600) t_done = load.now() time.sleep(observe) t_end = load.now() with load.lock: events = list(load.events) during = [e for e in events if t_action <= e[0] < t_done] return { 'action': Path(action).name, 'action_exit_code': proc.returncode, 'action_output': proc.stdout.strip().splitlines()[-30:], 'action_seconds': round(t_done - t_action, 2), 'max_gap_without_success_s': max_gap(events, t_action, t_done), 'baseline': summarize([e for e in events if e[0] < t_action], 0, t_action), 'during_action': summarize(during, t_action, t_done), 'after_action': summarize([e for e in events if t_done <= e[0] < t_end], t_done, t_end), 'versions_before': dict(Counter(e[4] for e in events if e[2] and e[0] < t_action)), 'versions_during': dict(Counter(e[4] for e in during if e[2])), 'versions_after': dict(Counter(e[4] for e in events if e[2] and t_done <= e[0] < t_end)), 'marks': {'action_start': round(t_action, 2), 'action_end': round(t_done, 2)}, } def main(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument('--base-url', default=None) p.add_argument('--mode', choices=['failover', 'action'], default='failover') p.add_argument('--action', help='script to run under load in --mode action') p.add_argument('--concurrency', type=int, default=4) p.add_argument('--baseline', type=float, default=10) p.add_argument('--observe', type=float, default=15) p.add_argument('--settle', type=float, default=5) p.add_argument('--stop-timeout', type=int, default=9) p.add_argument('--kill', action='store_true', help='docker kill (SIGKILL, crash without drain) instead of docker stop') p.add_argument('--output', help='JSON result file (no tokens)') args = p.parse_args() check(args.mode != 'action' or args.action, '--action is required in action mode') c = Client(args.base_url) alice, bob, _, chat = c.bootstrap() replicas = api_containers() if args.mode == 'failover': check(len(replicas) >= 2, 'need at least two running api containers') load = Load(args.base_url, args.concurrency) load.start(alice, chat) runs = [] try: if args.mode == 'failover': for name, cid in replicas: print('%s %s (%s) under load...' % ('killing' if args.kill else 'stopping', name, cid), flush=True) runs.append(failover_run(load, name, cid, args.baseline, args.observe, args.settle, args.stop_timeout, args.kill)) r = runs[-1] print(' errors=%d gap=%.2fs first_ok_after_exit=%s rps %.1f -> %.1f -> %.1f rejoin=%ss' % ( r['during_stop_and_outage']['errors'], r['max_gap_without_success_s'], r['first_success_after_container_exit_s'], r['baseline']['rps_ok'], r['during_stop_and_outage']['rps_ok'], r['after_restore']['rps_ok'], r['rejoin_after_docker_start_s']), flush=True) else: print('running %s under load...' % args.action, flush=True) runs.append(action_run(load, args.action, args.baseline, args.observe)) finally: load.finish() end = load.now() confirmed = dict(load.confirmed) stored = [m for m in history(c, bob, chat) if m['text'].startswith('series-')] counts = Counter(m['text'] for m in stored) duplicates = sum(n - 1 for n in counts.values() if n > 1) stored_ids = {m['id'] for m in stored} missing = [mid for mid in confirmed if mid not in stored_ids] integrity = {'confirmed_writes': len(confirmed), 'stored_messages': len(stored), 'duplicates': duplicates, 'missing_confirmed': len(missing)} report = { 'mode': args.mode, 'stop_method': 'docker kill (SIGKILL)' if args.kill else 'docker stop --time %d (SIGTERM, drain)' % args.stop_timeout, 'concurrency': args.concurrency, 'workload': 'closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest', 'replicas_before': [name for name, _ in replicas], 'runs': runs, 'total': summarize(load.events, 0, end), 'integrity': integrity, 'timeline_per_second': timeline(load, end), } text = json.dumps(report, ensure_ascii=False, indent=2) if args.output: out = Path(args.output) out.parent.mkdir(parents=True, exist_ok=True) out.write_text(text + '\n', encoding='utf-8') print(json.dumps({'total': report['total'], 'integrity': integrity}, ensure_ascii=False)) check(duplicates == 0 and not missing, 'duplicates or lost confirmed messages: %s' % integrity) if args.mode == 'action': check(runs[0]['action_exit_code'] == 0, 'action script failed') print('PASS: %s, %d confirmed writes, no duplicates, nothing lost' % (args.mode, len(confirmed))) if __name__ == '__main__': run(main)