Files
mtusi2026lab7/scripts/test_resilience.py
T

124 lines
4.6 KiB
Python

"""Run the unchanged Make targets, with Docker actions executed by the host."""
import argparse
import ssl
import subprocess
import tempfile
import time
import urllib.request
from pathlib import Path
PROJECT = "mtusi-lab7-grade3"
COMPOSE = ["docker", "compose", "--project-name", PROJECT]
STORAGE = ("postgres", "s3", "broker")
APP = ("api1", "api2", "worker")
def compose(*args):
subprocess.run([*COMPOSE, *args], check=True)
def snapshot(*services):
for service in services:
ident = subprocess.check_output([*COMPOSE, "ps", "-aq", service], text=True).strip()
state = subprocess.check_output([
"docker", "inspect", "--format",
"{{.Id}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}",
ident,
], text=True).strip()
print(f"{service} {state}", flush=True)
def ready():
context = ssl.create_default_context(cafile="secrets/ca/public.crt")
for attempt in range(40):
try:
with urllib.request.urlopen("https://localhost:8443/health/ready", context=context, timeout=2):
return
except OSError:
if attempt == 39:
raise
time.sleep(0.5)
def broker_ready():
identifier = subprocess.check_output([*COMPOSE, "ps", "-q", "broker"], text=True).strip()
for attempt in range(45):
status = subprocess.check_output(["docker", "inspect", "--format", "{{.State.Health.Status}}", identifier],
text=True).strip()
if status == "healthy":
return
if attempt == 44:
raise TimeoutError("RabbitMQ did not become healthy")
time.sleep(1)
def scenario(mode):
hook = "scripts/restart.sh" if mode == "persistence" else "scripts/stop-one.sh"
with tempfile.TemporaryDirectory(prefix="hook-", dir="secrets") as directory:
bridge = Path(directory).resolve()
process = subprocess.Popen([
*COMPOSE, "run", "--rm", "--no-deps", "-T",
"-v", f"{bridge}:/hook", "-e", "LAB7_HOOK_DIR=/hook",
"tools", "make", f"test-{mode}", f"ACTION_SCRIPT={hook}",
])
target = None
try:
deadline = time.monotonic() + 180
while process.poll() is None:
if time.monotonic() > deadline:
raise TimeoutError("Resilience test exceeded 180 seconds")
if target is None and (bridge / "request").is_file():
target = (bridge / "request").read_text().strip()
allowed = {"restart"} if mode == "persistence" else {"api1", "api2"}
if target not in allowed:
(bridge / "result").write_bytes(b"2\n")
raise RuntimeError("Invalid Compose hook request")
print(f"Hook accepted for project {PROJECT}: {target}", flush=True)
if target == "restart":
snapshot(*APP, *STORAGE)
compose("restart", "-t", "3", *STORAGE)
compose("restart", "-t", "3", *APP)
snapshot(*APP, *STORAGE)
else:
compose("stop", "-t", "1", target)
(bridge / "result.tmp").write_bytes(b"0\n")
(bridge / "result.tmp").replace(bridge / "result")
time.sleep(0.1)
if process.wait() != 0 or target is None:
raise RuntimeError(f"Public {mode} test failed")
if target in {"api1", "api2"}:
snapshot("api1", "api2")
return target
finally:
if process.poll() is None:
process.terminate()
process.wait(timeout=10)
if target in {"api1", "api2"}:
compose("start", target)
ready()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("mode", choices=("persistence", "failover", "all"))
mode = parser.parse_args().mode
ready()
if mode in {"persistence", "all"}:
scenario("persistence")
if mode in {"failover", "all"}:
broker_ready()
targets = set()
for _ in range(5):
targets.add(scenario("failover"))
if targets == {"api1", "api2"}:
print("PASS: public failover test observed both stopped replicas")
break
# One additional request advances round-robin before the next test.
ready()
else:
raise RuntimeError("Public tests did not exercise both separate replicas")
if __name__ == "__main__":
main()