forked from mtusi2026/mtusi2026lab3
72 lines
2.6 KiB
Python
72 lines
2.6 KiB
Python
"""Check LB failover when api1 still runs but cannot reach its stores."""
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure, check, run # noqa: E402
|
|
|
|
|
|
def docker(*args):
|
|
return subprocess.run(["docker", *args], cwd=ROOT, check=True,
|
|
capture_output=True, text=True).stdout.strip()
|
|
|
|
|
|
def compose(*args):
|
|
return docker("compose", "--env-file", ".env", *args)
|
|
|
|
|
|
def details(container):
|
|
return json.loads(docker("inspect", container))[0]
|
|
|
|
|
|
def wait_health(container, expected, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
if details(container)["State"]["Health"]["Status"] == expected:
|
|
return
|
|
time.sleep(1)
|
|
raise Failure("api1 did not become " + expected)
|
|
|
|
|
|
def main():
|
|
client = Client(timeout=8)
|
|
seen = set()
|
|
for _ in range(20):
|
|
seen.add(client.call("GET", "/health/ready", schema="Health")
|
|
["headers"]["X-Instance-Id"])
|
|
check(seen == {"api-1", "api-2"}, "both instances must be ready initially")
|
|
actor, _, _, chat = client.bootstrap()
|
|
container = compose("ps", "-q", "api1")
|
|
check(bool(container), "api1 container is missing")
|
|
network = next((name for name in details(container)["NetworkSettings"]["Networks"]
|
|
if name.endswith("_backend")), None)
|
|
check(network is not None, "api1 backend network is missing")
|
|
disconnected = False
|
|
try:
|
|
docker("network", "disconnect", network, container)
|
|
disconnected = True
|
|
wait_health(container, "unhealthy")
|
|
for _ in range(20):
|
|
for path, schema in (("/api/v1/users/me", "User"),
|
|
("/health/ready", "Health")):
|
|
response = client.call("GET", path, actor=actor, schema=schema)
|
|
check(response["headers"]["X-Instance-Id"] == "api-2",
|
|
path + " was not served by healthy api2")
|
|
for number in range(10):
|
|
response = client.message(actor, chat, "unready check " + str(number))
|
|
check(response["headers"]["X-Instance-Id"] == "api-2",
|
|
"message was not accepted by healthy api2")
|
|
print("PASS: api1 unhealthy with frontend connection; 40 GETs and 10 POSTs served by api2")
|
|
finally:
|
|
if disconnected:
|
|
docker("network", "connect", "--alias", "api1", network, container)
|
|
wait_health(container, "healthy")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|