forked from mtusi2026/mtusi2026lab7
124 lines
4.6 KiB
Python
124 lines
4.6 KiB
Python
"""Run the unchanged Make targets, with Docker actions executed by the host."""
|
|
import argparse
|
|
import ssl
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
PROJECT = "mtusi-lab7-grade3"
|
|
COMPOSE = ["docker", "compose", "--project-name", PROJECT]
|
|
STORAGE = ("postgres", "s3", "broker")
|
|
APP = ("api1", "api2", "worker")
|
|
|
|
|
|
def compose(*args):
|
|
subprocess.run([*COMPOSE, *args], check=True)
|
|
|
|
|
|
def snapshot(*services):
|
|
for service in services:
|
|
ident = subprocess.check_output([*COMPOSE, "ps", "-aq", service], text=True).strip()
|
|
state = subprocess.check_output([
|
|
"docker", "inspect", "--format",
|
|
"{{.Id}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}",
|
|
ident,
|
|
], text=True).strip()
|
|
print(f"{service} {state}", flush=True)
|
|
|
|
|
|
def ready():
|
|
context = ssl.create_default_context(cafile="secrets/ca/public.crt")
|
|
for attempt in range(40):
|
|
try:
|
|
with urllib.request.urlopen("https://localhost:8443/health/ready", context=context, timeout=2):
|
|
return
|
|
except OSError:
|
|
if attempt == 39:
|
|
raise
|
|
time.sleep(0.5)
|
|
|
|
|
|
def broker_ready():
|
|
identifier = subprocess.check_output([*COMPOSE, "ps", "-q", "broker"], text=True).strip()
|
|
for attempt in range(45):
|
|
status = subprocess.check_output(["docker", "inspect", "--format", "{{.State.Health.Status}}", identifier],
|
|
text=True).strip()
|
|
if status == "healthy":
|
|
return
|
|
if attempt == 44:
|
|
raise TimeoutError("RabbitMQ did not become healthy")
|
|
time.sleep(1)
|
|
|
|
|
|
def scenario(mode):
|
|
hook = "scripts/restart.sh" if mode == "persistence" else "scripts/stop-one.sh"
|
|
with tempfile.TemporaryDirectory(prefix="hook-", dir="secrets") as directory:
|
|
bridge = Path(directory).resolve()
|
|
process = subprocess.Popen([
|
|
*COMPOSE, "run", "--rm", "--no-deps", "-T",
|
|
"-v", f"{bridge}:/hook", "-e", "LAB7_HOOK_DIR=/hook",
|
|
"tools", "make", f"test-{mode}", f"ACTION_SCRIPT={hook}",
|
|
])
|
|
target = None
|
|
try:
|
|
deadline = time.monotonic() + 180
|
|
while process.poll() is None:
|
|
if time.monotonic() > deadline:
|
|
raise TimeoutError("Resilience test exceeded 180 seconds")
|
|
if target is None and (bridge / "request").is_file():
|
|
target = (bridge / "request").read_text().strip()
|
|
allowed = {"restart"} if mode == "persistence" else {"api1", "api2"}
|
|
if target not in allowed:
|
|
(bridge / "result").write_bytes(b"2\n")
|
|
raise RuntimeError("Invalid Compose hook request")
|
|
print(f"Hook accepted for project {PROJECT}: {target}", flush=True)
|
|
if target == "restart":
|
|
snapshot(*APP, *STORAGE)
|
|
compose("restart", "-t", "3", *STORAGE)
|
|
compose("restart", "-t", "3", *APP)
|
|
snapshot(*APP, *STORAGE)
|
|
else:
|
|
compose("stop", "-t", "1", target)
|
|
(bridge / "result.tmp").write_bytes(b"0\n")
|
|
(bridge / "result.tmp").replace(bridge / "result")
|
|
time.sleep(0.1)
|
|
if process.wait() != 0 or target is None:
|
|
raise RuntimeError(f"Public {mode} test failed")
|
|
if target in {"api1", "api2"}:
|
|
snapshot("api1", "api2")
|
|
return target
|
|
finally:
|
|
if process.poll() is None:
|
|
process.terminate()
|
|
process.wait(timeout=10)
|
|
if target in {"api1", "api2"}:
|
|
compose("start", target)
|
|
ready()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("mode", choices=("persistence", "failover", "all"))
|
|
mode = parser.parse_args().mode
|
|
ready()
|
|
if mode in {"persistence", "all"}:
|
|
scenario("persistence")
|
|
if mode in {"failover", "all"}:
|
|
broker_ready()
|
|
targets = set()
|
|
for _ in range(5):
|
|
targets.add(scenario("failover"))
|
|
if targets == {"api1", "api2"}:
|
|
print("PASS: public failover test observed both stopped replicas")
|
|
break
|
|
# One additional request advances round-robin before the next test.
|
|
ready()
|
|
else:
|
|
raise RuntimeError("Public tests did not exercise both separate replicas")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|