import importlib import json import subprocess import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) Client = importlib.import_module("client").Client def command(*args): return subprocess.run(args, check=True, capture_output=True, text=True).stdout client = Client("https://localhost:18443", str(ROOT / "secrets/pki/operator/ca.crt")) alice, _, _, chat = client.bootstrap() containers = json.loads( command("docker", "inspect", *command("docker", "compose", "ps", "-q", "api").split()) ) results = [] for item in containers: target = item["Config"]["Hostname"] attachment, content = client.attachment(alice, chat) client.wait_ready(alice, attachment) before = client.message(alice, chat, "Before replica stop")["json"] started = time.monotonic() failures = 0 success = 0 instances = set() try: command("docker", "stop", "-t", "7", item["Id"]) deadline = time.monotonic() + 20 first = None while success < 30 and time.monotonic() < deadline: try: response = client.message(alice, chat, "After replica stop") success += 1 instances.add(response["headers"]["X-Instance-Id"]) first = first or time.monotonic() - started except AssertionError: failures += 1 time.sleep(0.2) if success != 30 or target in instances or len(instances) != 1: raise RuntimeError("Replica failover did not isolate stopped instance") data = client.call( "GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=alice, schema="MessagePage" )["json"] if before["id"] not in [row["id"] for row in data["items"]]: raise RuntimeError("Confirmed message lost") if ( client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=alice)[ "bytes" ] != content ): raise RuntimeError("Attachment changed") results.append( { "target": target, "successful_writes": success, "failed_attempts": failures, "first_recovery_seconds": round(first, 3), "message_attachment_session_preserved": True, } ) finally: command("docker", "start", item["Id"]) command("scripts/lb-wait.sh") print(json.dumps(results, indent=2)) (ROOT / "evidence/security/failover-each.json").write_text(json.dumps(results, indent=2) + "\n")