forked from mtusi2026/mtusi2026lab7
73 lines
2.6 KiB
Python
73 lines
2.6 KiB
Python
import importlib
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
Client = importlib.import_module("client").Client
|
|
|
|
|
|
def command(*args):
|
|
return subprocess.run(args, check=True, capture_output=True, text=True).stdout
|
|
|
|
|
|
client = Client("https://localhost:18443", str(ROOT / "secrets/pki/operator/ca.crt"))
|
|
alice, _, _, chat = client.bootstrap()
|
|
containers = json.loads(
|
|
command("docker", "inspect", *command("docker", "compose", "ps", "-q", "api").split())
|
|
)
|
|
results = []
|
|
for item in containers:
|
|
target = item["Config"]["Hostname"]
|
|
attachment, content = client.attachment(alice, chat)
|
|
client.wait_ready(alice, attachment)
|
|
before = client.message(alice, chat, "Before replica stop")["json"]
|
|
started = time.monotonic()
|
|
failures = 0
|
|
success = 0
|
|
instances = set()
|
|
try:
|
|
command("docker", "stop", "-t", "7", item["Id"])
|
|
deadline = time.monotonic() + 20
|
|
first = None
|
|
while success < 30 and time.monotonic() < deadline:
|
|
try:
|
|
response = client.message(alice, chat, "After replica stop")
|
|
success += 1
|
|
instances.add(response["headers"]["X-Instance-Id"])
|
|
first = first or time.monotonic() - started
|
|
except AssertionError:
|
|
failures += 1
|
|
time.sleep(0.2)
|
|
if success != 30 or target in instances or len(instances) != 1:
|
|
raise RuntimeError("Replica failover did not isolate stopped instance")
|
|
data = client.call(
|
|
"GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=alice, schema="MessagePage"
|
|
)["json"]
|
|
if before["id"] not in [row["id"] for row in data["items"]]:
|
|
raise RuntimeError("Confirmed message lost")
|
|
if (
|
|
client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=alice)[
|
|
"bytes"
|
|
]
|
|
!= content
|
|
):
|
|
raise RuntimeError("Attachment changed")
|
|
results.append(
|
|
{
|
|
"target": target,
|
|
"successful_writes": success,
|
|
"failed_attempts": failures,
|
|
"first_recovery_seconds": round(first, 3),
|
|
"message_attachment_session_preserved": True,
|
|
}
|
|
)
|
|
finally:
|
|
command("docker", "start", item["Id"])
|
|
command("scripts/lb-wait.sh")
|
|
print(json.dumps(results, indent=2))
|
|
(ROOT / "evidence/security/failover-each.json").write_text(json.dumps(results, indent=2) + "\n")
|