Files
mtusi2026lab5/scripts/verify_stack.py
T

72 lines
3.5 KiB
Python

"""Record real restart scope and exercise failover of both physical API containers."""
import json
import os
import subprocess
from pathlib import Path
def run(args):
result = subprocess.run(args, text=True, capture_output=True, timeout=180)
if result.returncode:
raise RuntimeError("Verification failed:\n" + result.stdout + result.stderr[-2000:])
return result
def snapshot():
ids = run(["docker", "compose", "ps", "-aq"]).stdout.split()
items = json.loads(run(["docker", "inspect", *ids]).stdout)
return [
{"id": item["Id"], "service": item["Config"]["Labels"]["com.docker.compose.service"],
"started_at": item["State"]["StartedAt"], "status": item["State"]["Status"],
"mounts": [{"type": mount["Type"], "name": mount.get("Name"), "destination": mount["Destination"]}
for mount in item["Mounts"] if mount["Type"] == "volume"]}
for item in items
]
def main():
base = os.getenv("BASE_URL", "http://127.0.0.1:8085")
before = snapshot()
result = run(["python3", "tests/resilience.py", "--mode", "persistence", "--action-script", "scripts/restart.sh", "--base-url", base])
Path("evidence/persistence-final.txt").write_text(result.stdout)
after = snapshot()
changed = []
by_id = {item["id"]: item for item in before}
for item in after:
old = by_id.get(item["id"])
if old and old["started_at"] != item["started_at"]:
changed.append(item["service"])
required = {"postgres", "cache", "broker", "storage", "s3", "api", "lb", "worker", "dispatcher"}
assert required <= set(changed), (required, changed)
assert {json.dumps(m, sort_keys=True) for i in before for m in i["mounts"]} == {json.dumps(m, sort_keys=True) for i in after for m in i["mounts"]}
Path("evidence/persistence-runtime.json").write_text(json.dumps({"before": before, "after": after, "restarted_services": changed, "volumes_unchanged": True, "passed": True}, indent=2))
actual_api = {item["id"] for item in after if item["service"] == "api"}
assert len(actual_api) >= 2
stopped = set()
attempts = []
try:
for _ in range(6):
result = run(["python3", "tests/resilience.py", "--mode", "failover", "--action-script", "scripts/stop-one.sh", "--base-url", base])
receipt = json.loads(Path("evidence/last-stopped.json").read_text())
assert receipt["stopped_container"] in actual_api
state = json.loads(run(["docker", "inspect", receipt["stopped_container"]]).stdout)[0]
assert not state["State"]["Running"]
summaries = [json.loads(line) for line in result.stdout.splitlines() if line.startswith("{")]
attempts.append({"stopped": receipt, "summary": summaries, "stdout": result.stdout})
stopped.add(receipt["stopped_container"])
run(["docker", "compose", "start", "api"])
run(["docker", "compose", "up", "-d", "--no-recreate", "--wait", "api", "lb"])
if stopped == actual_api:
break
assert stopped == actual_api, "Both physical replicas were not covered"
finally:
run(["docker", "compose", "start", "api"])
Path("evidence/failover-final.json").write_text(json.dumps({"actual_api_container_ids": sorted(actual_api), "covered_container_ids": sorted(stopped), "attempts": attempts, "passed": True}, indent=2))
print("PASS: full restart and failover of both physical API replicas")
if __name__ == "__main__":
main()