#!/usr/bin/env python3 import argparse import json import os import subprocess import sys import threading import time import uuid from concurrent.futures import ThreadPoolExecutor from pathlib import Path from urllib.parse import quote ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure BASE_URL = os.environ.get('BASE_URL', 'http://lb:8080') def docker(*args): return subprocess.run(['docker', *args], check=True, capture_output=True, text=True).stdout.strip() def api_containers(): return [cid[:12] for cid in docker('compose', 'ps', '-q', 'api').split()] def describe(cid): return docker('inspect', '-f', '{{.Name}} {{.State.Status}} image={{slice .Image 7 19}}', cid) def wait_healthy(cid, timeout=60): deadline = time.monotonic() + timeout while time.monotonic() < deadline: if docker('inspect', '-f', '{{.State.Health.Status}}', cid) == 'healthy': return time.sleep(0.2) raise SystemExit(cid + ' did not become healthy') class Load: def __init__(self, client, actor, chat, workers): self.client, self.actor, self.chat = client, actor, chat self.workers = workers self.lock = threading.Lock() self.events = [] self.confirmed = [] self.stop_flag = threading.Event() self.threads = [threading.Thread(target=self.worker, args=(i,)) for i in range(workers)] self.started = None def worker(self, index): local = Client(BASE_URL, timeout=2) key = uuid.uuid4().hex while not self.stop_flag.is_set(): try: response = local.message(self.actor, self.chat, 'load ' + key, key=key) with self.lock: self.events.append((time.monotonic(), True, response['headers']['X-Instance-Id'])) self.confirmed.append(response['json']) key = uuid.uuid4().hex except Failure: with self.lock: self.events.append((time.monotonic(), False, None)) time.sleep(0.05) def __enter__(self): self.started = time.monotonic() for thread in self.threads: thread.start() return self def __exit__(self, *exc): self.stop_flag.set() for thread in self.threads: thread.join() def window(self, start, end): with self.lock: events = [e for e in self.events if start <= e[0] < end] ok = sorted(e[0] for e in events if e[1]) points = [start, *ok, end] return { 'seconds': round(end - start, 3), 'successful_writes': len(ok), 'failed_attempts': sum(1 for e in events if not e[1]), 'successful_rps': round(len(ok) / (end - start), 1), 'max_write_gap_seconds': round(max(b - a for a, b in zip(points, points[1:])), 3), 'instances': sorted({e[2] for e in events if e[1]}), } def last_seen(self, instance, start, end): with self.lock: times = [e[0] for e in self.events if e[1] and e[2] == instance and start <= e[0] < end] return max(times) if times else None def first_seen_after(self, instance, after): with self.lock: times = [e[0] for e in self.events if e[1] and e[2] == instance and e[0] >= after] return min(times) if times else None def verify_history(client, actor, chat, confirmed): path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100' items = [] while True: page = client.call('GET', path, actor=actor, schema='MessagePage')['json'] items.extend(page['items']) if page['next_cursor'] is None: break path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='') texts = [m['text'] for m in items] stored = {m['id']: m for m in items} return { 'confirmed': len(confirmed), 'stored': len(items), 'lost_confirmed': sum(1 for m in confirmed if stored.get(m['id']) != m), 'duplicates': len(texts) - len(set(texts)), } def failover(load, args): results = [] with load: time.sleep(args.warmup) for cid in api_containers(): t0 = time.monotonic() docker('kill' if args.kill else 'stop', cid) stopped = time.monotonic() time.sleep(args.outage) outage = load.window(t0, time.monotonic()) restore = time.monotonic() docker('start', cid) wait_healthy(cid) healthy = time.monotonic() time.sleep(args.warmup) rejoined = load.first_seen_after(cid, restore) last = load.last_seen(cid, t0, restore) results.append({ 'container': describe(cid), 'action': 'docker kill (SIGKILL)' if args.kill else 'docker stop (SIGTERM, graceful)', 'before': load.window(t0 - args.warmup, t0), 'stop_command_seconds': round(stopped - t0, 3), 'target_last_success_after_stop_seconds': round(last - t0, 3) if last else None, 'outage_window': outage, 'restore_until_healthy_seconds': round(healthy - restore, 3), 'restore_until_serving_seconds': round(rejoined - restore, 3) if rejoined else None, 'after_restore': load.window(healthy, time.monotonic()), }) return {'scenario': 'sequential failover of each api replica', 'replicas': results} def rolling(load, args): old = api_containers() before = [describe(cid) for cid in old] steps = [] with load: time.sleep(args.warmup) t_start = time.monotonic() docker('compose', 'run', '--rm', '--no-deps', 'migrate') for cid in old: t0 = time.monotonic() docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=' + str(len(old) + 1), '--wait', 'api') added = time.monotonic() docker('stop', cid) docker('rm', cid) steps.append({'replaced': cid, 'new_replica_ready_seconds': round(added - t0, 3), 'step_seconds': round(time.monotonic() - t0, 3)}) time.sleep(args.warmup) total = load.window(t_start, time.monotonic()) return { 'scenario': 'rolling replacement: migrate once, then add new replica, wait healthy, stop old one', 'before': before, 'after': [describe(cid) for cid in api_containers()], 'steps': steps, 'during_update': total, } def idempotency(client, actor, chat, args): key = uuid.uuid4().hex text = 'parallel ' + key def send(_): local = Client(BASE_URL, timeout=10) response = local.message(actor, chat, text, key=key) return response['json']['id'], response['headers']['X-Instance-Id'] with ThreadPoolExecutor(max_workers=args.parallel) as pool: results = list(pool.map(send, range(args.parallel))) path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100' stored = [m for m in client.call('GET', path, actor=actor, schema='MessagePage')['json']['items'] if m['text'] == text] return { 'scenario': 'parallel POST with one Idempotency-Key through LB', 'requests': args.parallel, 'all_201': True, 'distinct_message_ids': len({r[0] for r in results}), 'instances': sorted({r[1] for r in results}), 'messages_in_history': len(stored), } def main(): parser = argparse.ArgumentParser(description='Lab3 level-4 experiments under load through LB') parser.add_argument('scenario', choices=['failover', 'rolling', 'idempotency']) parser.add_argument('--workers', type=int, default=4) parser.add_argument('--warmup', type=float, default=5) parser.add_argument('--outage', type=float, default=10) parser.add_argument('--parallel', type=int, default=20) parser.add_argument('--kill', action='store_true') parser.add_argument('--output') args = parser.parse_args() client = Client(BASE_URL) alice, bob, _, chat = client.bootstrap() scenarios = {'failover': failover, 'rolling': rolling} if args.scenario in scenarios: load = Load(client, alice, chat, args.workers) report = scenarios[args.scenario](load, args) report['history'] = verify_history(client, bob, chat, load.confirmed) report['load'] = {'workers': args.workers, 'client_timeout_seconds': 2, 'retry': 'same Idempotency-Key until success'} else: report = idempotency(client, alice, chat, args) report['date'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) text = json.dumps(report, ensure_ascii=False, indent=2) if args.output: Path(args.output).parent.mkdir(parents=True, exist_ok=True) Path(args.output).write_text(text + '\n') print(text) if __name__ == '__main__': main()