Files
mtusi2026lab7/scripts/security_failover.py
T

73 lines
2.6 KiB
Python

import importlib
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
Client = importlib.import_module("client").Client
def command(*args):
return subprocess.run(args, check=True, capture_output=True, text=True).stdout
client = Client("https://localhost:18443", str(ROOT / "secrets/pki/operator/ca.crt"))
alice, _, _, chat = client.bootstrap()
containers = json.loads(
command("docker", "inspect", *command("docker", "compose", "ps", "-q", "api").split())
)
results = []
for item in containers:
target = item["Config"]["Hostname"]
attachment, content = client.attachment(alice, chat)
client.wait_ready(alice, attachment)
before = client.message(alice, chat, "Before replica stop")["json"]
started = time.monotonic()
failures = 0
success = 0
instances = set()
try:
command("docker", "stop", "-t", "7", item["Id"])
deadline = time.monotonic() + 20
first = None
while success < 30 and time.monotonic() < deadline:
try:
response = client.message(alice, chat, "After replica stop")
success += 1
instances.add(response["headers"]["X-Instance-Id"])
first = first or time.monotonic() - started
except AssertionError:
failures += 1
time.sleep(0.2)
if success != 30 or target in instances or len(instances) != 1:
raise RuntimeError("Replica failover did not isolate stopped instance")
data = client.call(
"GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=alice, schema="MessagePage"
)["json"]
if before["id"] not in [row["id"] for row in data["items"]]:
raise RuntimeError("Confirmed message lost")
if (
client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=alice)[
"bytes"
]
!= content
):
raise RuntimeError("Attachment changed")
results.append(
{
"target": target,
"successful_writes": success,
"failed_attempts": failures,
"first_recovery_seconds": round(first, 3),
"message_attachment_session_preserved": True,
}
)
finally:
command("docker", "start", item["Id"])
command("scripts/lb-wait.sh")
print(json.dumps(results, indent=2))
(ROOT / "evidence/security/failover-each.json").write_text(json.dumps(results, indent=2) + "\n")