#!/usr/bin/env python3 """Exercise a real Swarm worker loss while reading and writing through the LB.""" import argparse from concurrent.futures import ThreadPoolExecutor import json from pathlib import Path import statistics import subprocess import time import uuid from client import Client, Failure def tasks(): result = subprocess.check_output([ "docker", "exec", "lab3-manager", "docker", "service", "ps", "--filter", "desired-state=running", "--format", "{{.Node}} {{.CurrentState}}", "lab3app_api", ], text=True) return [line.split()[0] for line in result.splitlines() if "Running" in line] def main(): parser = argparse.ArgumentParser() parser.add_argument("--base-url", default="http://127.0.0.1:18080") parser.add_argument("--worker", choices=["lab3-worker1", "lab3-worker2"], default="lab3-worker1") parser.add_argument("--output", default="evidence/node_failover.json") args = parser.parse_args() client = Client(args.base_url, timeout=2) alice, bob, _, chat = client.bootstrap() before = tasks() assert len(before) == 2 and args.worker in before, before instances = { client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"] for _ in range(40) } assert len(instances) == 2, instances key = uuid.uuid4().hex body = "concurrent " + key with ThreadPoolExecutor(max_workers=16) as executor: results = list(executor.map(lambda _: client.message(alice, chat, body, key=key)["json"], range(16))) assert all(item == results[0] for item in results) subprocess.run(["docker", "stop", "-t", "1", args.worker], check=True, stdout=subprocess.DEVNULL) start = time.monotonic() successes = [] errors = 0 latencies = [] moved_at = None first_success = None pending_key = uuid.uuid4().hex pending_text = "node-fail " + pending_key while time.monotonic() - start < 75: try: response = client.message(alice, chat, pending_text, key=pending_key) successes.append(response["json"]) latencies.append(response["elapsed"] * 1000) if first_success is None: first_success = time.monotonic() - start client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=1", actor=bob, schema="MessagePage") pending_key = uuid.uuid4().hex pending_text = "node-fail " + pending_key except Failure: errors += 1 current = tasks() if len(current) == 2 and args.worker not in current and moved_at is None: moved_at = time.monotonic() - start if moved_at is not None and time.monotonic() - start > moved_at + 5: break time.sleep(0.15) assert moved_at is not None, tasks() assert successes, "no writes during node failure" page = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage")["json"] seen = {item["id"] for item in page["items"]} assert results[0]["id"] in seen and all(item["id"] in seen for item in successes) assert sum(item["text"] == body for item in page["items"]) == 1 report = { "before_nodes": before, "after_nodes": tasks(), "before_instance_count": len(instances), "concurrent_replays": len(results), "concurrent_message_count": 1, "successful_writes": len(successes), "failed_attempts": errors, "first_success_seconds": round(first_success, 3), "replacement_running_seconds": round(moved_at, 3), "confirmed_messages_preserved": True, "latency_p50_ms": round(statistics.median(latencies), 3), "latency_p95_ms": round(sorted(latencies)[int(0.95 * (len(latencies) - 1))], 3), } Path(args.output).write_text(json.dumps(report, indent=2) + "\n") print(json.dumps(report, indent=2)) if __name__ == "__main__": main()