#!/usr/bin/env python3 """Exercise a real Swarm worker loss while reading and writing through the LB.""" import argparse from collections import Counter from concurrent.futures import ThreadPoolExecutor import json from pathlib import Path import statistics import subprocess import time import uuid from urllib.parse import quote from client import Client, Failure def tasks(): result = subprocess.check_output([ "docker", "exec", "lab3-manager", "docker", "service", "ps", "--filter", "desired-state=running", "--format", "{{.ID}} {{.Node}} {{.CurrentState}}", "lab3app_api", ], text=True) return [{"task_id": parts[0], "node": parts[1]} for line in result.splitlines() if len(parts := line.split()) >= 3 and parts[2] == "Running"] def history(client, actor, chat): path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" items = [] for _ in range(100): page = client.call("GET", path, actor=actor, schema="MessagePage")["json"] items.extend(page["items"]) if page["next_cursor"] is None: return items path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100&cursor=" + quote(page["next_cursor"], safe="") raise AssertionError("history pagination did not terminate") def main(): parser = argparse.ArgumentParser() parser.add_argument("--base-url", default="http://127.0.0.1:18080") parser.add_argument("--worker", choices=["lab3-worker1", "lab3-worker2"], default="lab3-worker1") parser.add_argument("--output", default="evidence/node_failover.json") args = parser.parse_args() client = Client(args.base_url, timeout=2) alice, bob, _, chat = client.bootstrap() before = tasks() assert len(before) == 2 and args.worker in {task["node"] for task in before}, before instances = { client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"] for _ in range(40) } assert len(instances) == 2, instances key = uuid.uuid4().hex body = "concurrent " + key with ThreadPoolExecutor(max_workers=16) as executor: responses = list(executor.map(lambda _: client.message(alice, chat, body, key=key), range(16))) results = [response["json"] for response in responses] concurrent_instances = {response["headers"]["X-Instance-Id"] for response in responses} assert all(item == results[0] for item in results) assert len(concurrent_instances) >= 2, concurrent_instances start = time.monotonic() subprocess.run(["docker", "stop", "-t", "1", args.worker], check=True, stdout=subprocess.DEVNULL) stop_return_seconds = time.monotonic() - start successes = {} post_errors = 0 read_errors = 0 successful_reads = 0 latencies = [] moved_at = None first_success = None pending_key = uuid.uuid4().hex pending_text = "node-fail " + pending_key while time.monotonic() - start < 75: try: response = client.message(alice, chat, pending_text, key=pending_key) assert pending_key not in successes successes[pending_key] = response["json"] latest_id = response["json"]["id"] latencies.append(response["elapsed"] * 1000) if first_success is None: first_success = time.monotonic() - start pending_key = uuid.uuid4().hex pending_text = "node-fail " + pending_key except Failure: post_errors += 1 else: try: assert latest_id in {item["id"] for item in history(client, bob, chat)} successful_reads += 1 except Failure: read_errors += 1 current = tasks() if len(current) == 2 and args.worker not in {task["node"] for task in current} and moved_at is None: moved_at = time.monotonic() - start if moved_at is not None and time.monotonic() - start > moved_at + 5: break time.sleep(0.15) assert moved_at is not None, tasks() assert successes, "no writes during node failure" items = history(client, bob, chat) ids = Counter(item["id"] for item in items) texts = Counter(item["text"] for item in items) assert ids[results[0]["id"]] == 1 and texts[body] == 1 assert all(ids[item["id"]] == 1 and texts[item["text"]] == 1 for item in successes.values()) after_instances = { client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"] for _ in range(40) } assert after_instances - instances, (instances, after_instances) report = { "before_tasks": before, "after_tasks": tasks(), "before_instance_ids": sorted(instances), "after_instance_ids": sorted(after_instances), "concurrent_replays": len(results), "concurrent_message_count": 1, "concurrent_instance_ids": sorted(concurrent_instances), "successful_writes": len(successes), "successful_reads": successful_reads, "failed_post_attempts": post_errors, "failed_read_attempts": read_errors, "docker_stop_seconds": round(stop_return_seconds, 3), "first_success_seconds": round(first_success, 3), "replacement_running_seconds": round(moved_at, 3), "confirmed_messages_preserved": True, "latency_p50_ms": round(statistics.median(latencies), 3), "latency_p95_ms": round(sorted(latencies)[int(0.95 * (len(latencies) - 1))], 3), } Path(args.output).write_text(json.dumps(report, indent=2) + "\n") print(json.dumps(report, indent=2)) if __name__ == "__main__": main()