"""Record real restart scope and exercise failover of both physical API containers.""" import json import os import subprocess from pathlib import Path def run(args): result = subprocess.run(args, text=True, capture_output=True, timeout=180) if result.returncode: raise RuntimeError("Verification failed:\n" + result.stdout + result.stderr[-2000:]) return result def snapshot(): ids = run(["docker", "compose", "ps", "-aq"]).stdout.split() items = json.loads(run(["docker", "inspect", *ids]).stdout) return [ {"id": item["Id"], "service": item["Config"]["Labels"]["com.docker.compose.service"], "started_at": item["State"]["StartedAt"], "status": item["State"]["Status"], "mounts": [{"type": mount["Type"], "name": mount.get("Name"), "destination": mount["Destination"]} for mount in item["Mounts"] if mount["Type"] == "volume"]} for item in items ] def main(): base = os.getenv("BASE_URL", "http://127.0.0.1:8085") before = snapshot() result = run(["python3", "tests/resilience.py", "--mode", "persistence", "--action-script", "scripts/restart.sh", "--base-url", base]) Path("evidence/persistence-final.txt").write_text(result.stdout) after = snapshot() changed = [] by_id = {item["id"]: item for item in before} for item in after: old = by_id.get(item["id"]) if old and old["started_at"] != item["started_at"]: changed.append(item["service"]) required = {"postgres", "cache", "broker", "storage", "s3", "api", "lb", "worker", "dispatcher"} assert required <= set(changed), (required, changed) assert {json.dumps(m, sort_keys=True) for i in before for m in i["mounts"]} == {json.dumps(m, sort_keys=True) for i in after for m in i["mounts"]} Path("evidence/persistence-runtime.json").write_text(json.dumps({"before": before, "after": after, "restarted_services": changed, "volumes_unchanged": True, "passed": True}, indent=2)) actual_api = {item["id"] for item in after if item["service"] == "api"} assert len(actual_api) >= 2 stopped = set() attempts = [] try: for _ in range(6): result = run(["python3", "tests/resilience.py", "--mode", "failover", "--action-script", "scripts/stop-one.sh", "--base-url", base]) receipt = json.loads(Path("evidence/last-stopped.json").read_text()) assert receipt["stopped_container"] in actual_api state = json.loads(run(["docker", "inspect", receipt["stopped_container"]]).stdout)[0] assert not state["State"]["Running"] summaries = [json.loads(line) for line in result.stdout.splitlines() if line.startswith("{")] attempts.append({"stopped": receipt, "summary": summaries, "stdout": result.stdout}) stopped.add(receipt["stopped_container"]) run(["docker", "compose", "start", "api"]) run(["docker", "compose", "up", "-d", "--no-recreate", "--wait", "api", "lb"]) if stopped == actual_api: break assert stopped == actual_api, "Both physical replicas were not covered" finally: run(["docker", "compose", "start", "api"]) Path("evidence/failover-final.json").write_text(json.dumps({"actual_api_container_ids": sorted(actual_api), "covered_container_ids": sorted(stopped), "attempts": attempts, "passed": True}, indent=2)) print("PASS: full restart and failover of both physical API replicas") if __name__ == "__main__": main()