import os import subprocess import sys from common import BASE_URL, ROOT, api_containers, call, check, compose_output, docker, wait_balanced HOOK = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh") def ps(): return compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip() def aim(wanted): """Для 2 реплик: последним перед тестом должна ответить реплика из wanted.""" probe = call("GET", "/health/live", 200).instance if not any(container.startswith(probe) for container in wanted): call("GET", "/health/live", 200) # теперь последней ответила другая реплика def main(): containers = api_containers() check(len(containers) >= 2, "need at least 2 running api replicas") print(f"hook: {HOOK.relative_to(ROOT)}; api replicas: {[c[:12] for c in containers]}") stopped_once = set() for attempt in range(1, 7): wait_balanced(len(containers)) if len(containers) == 2: aim(set(containers) - stopped_once) print(f"=== run {attempt}: api containers before") print(ps()) env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "failover", "--action-script", str(HOOK), "--base-url", BASE_URL], cwd=ROOT, capture_output=True, env=env) print((result.stdout + result.stderr).decode("utf-8", "replace").strip()) check(result.returncode == 0, "public failover test failed") print("api containers after") print(ps()) stopped = [container for container in containers if container not in api_containers()] check(len(stopped) == 1, f"expected exactly one stopped api container, got {len(stopped)}") print(f"the hook stopped {stopped[0][:12]} (X-Instance-Id of that replica = container ID); " f"other api containers kept running, postgres/redis/lb were not touched") stopped_once.add(stopped[0]) docker("start", stopped[0]) wait_balanced(len(containers)) print(f"restored: docker start {stopped[0][:12]} -> healthy and back in HAProxy") if stopped_once == set(containers): break check(stopped_once == set(containers), "not every replica was stopped by the public test") print(f"PASS: public failover test passed {attempt} times; every api replica was stopped once and restored") if __name__ == "__main__": main()