forked from mtusi2026/mtusi2026lab7
60 lines
2.9 KiB
Python
60 lines
2.9 KiB
Python
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
from common import (BASE_URL, CA_FILE, ROOT, api_containers, call, check, compose_output, docker, instance_aliases,
|
|
wait_balanced)
|
|
|
|
HOOK = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh")
|
|
|
|
|
|
def ps():
|
|
return compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip()
|
|
|
|
|
|
def aim(wanted):
|
|
"""Для 2 реплик: последним перед тестом должна ответить реплика из wanted.
|
|
|
|
lab7: X-Instance-Id — случайный alias, а не ID контейнера; сопоставляем через instance_aliases()."""
|
|
aliases = {alias for container, alias in instance_aliases().items() if container in wanted}
|
|
probe = call("GET", "/health/live", 200).instance
|
|
if probe not in aliases:
|
|
call("GET", "/health/live", 200) # теперь последней ответила другая реплика
|
|
|
|
|
|
def main():
|
|
containers = api_containers()
|
|
check(len(containers) >= 2, "need at least 2 running api replicas")
|
|
print(f"hook: {HOOK.relative_to(ROOT)}; api replicas: {[c[:12] for c in containers]}")
|
|
stopped_once = set()
|
|
for attempt in range(1, 7):
|
|
wait_balanced(len(containers))
|
|
if len(containers) == 2:
|
|
aim(set(containers) - stopped_once)
|
|
print(f"=== run {attempt}: api containers before")
|
|
print(ps())
|
|
env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8")
|
|
result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "failover",
|
|
"--action-script", str(HOOK), "--base-url", BASE_URL, "--ca-file", CA_FILE],
|
|
cwd=ROOT, capture_output=True, env=env)
|
|
print((result.stdout + result.stderr).decode("utf-8", "replace").strip())
|
|
check(result.returncode == 0, "public failover test failed")
|
|
print("api containers after")
|
|
print(ps())
|
|
stopped = [container for container in containers if container not in api_containers()]
|
|
check(len(stopped) == 1, f"expected exactly one stopped api container, got {len(stopped)}")
|
|
print(f"the hook stopped {stopped[0][:12]} (the replica behind TARGET_INSTANCE_ID, mapped from its alias); "
|
|
f"other api containers kept running, postgres/redis/lb were not touched")
|
|
stopped_once.add(stopped[0])
|
|
docker("start", stopped[0])
|
|
wait_balanced(len(containers))
|
|
print(f"restored: docker start {stopped[0][:12]} -> healthy and back in HAProxy")
|
|
if stopped_once == set(containers):
|
|
break
|
|
check(stopped_once == set(containers), "not every replica was stopped by the public test")
|
|
print(f"PASS: public failover test passed {attempt} times; every api replica was stopped once and restored")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|