Files
mtusi2026lab3/student-tests/verify_unready_backend.py
T

72 lines
2.6 KiB
Python

"""Check LB failover when api1 still runs but cannot reach its stores."""
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check, run # noqa: E402
def docker(*args):
return subprocess.run(["docker", *args], cwd=ROOT, check=True,
capture_output=True, text=True).stdout.strip()
def compose(*args):
return docker("compose", "--env-file", ".env", *args)
def details(container):
return json.loads(docker("inspect", container))[0]
def wait_health(container, expected, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if details(container)["State"]["Health"]["Status"] == expected:
return
time.sleep(1)
raise Failure("api1 did not become " + expected)
def main():
client = Client(timeout=8)
seen = set()
for _ in range(20):
seen.add(client.call("GET", "/health/ready", schema="Health")
["headers"]["X-Instance-Id"])
check(seen == {"api-1", "api-2"}, "both instances must be ready initially")
actor, _, _, chat = client.bootstrap()
container = compose("ps", "-q", "api1")
check(bool(container), "api1 container is missing")
network = next((name for name in details(container)["NetworkSettings"]["Networks"]
if name.endswith("_backend")), None)
check(network is not None, "api1 backend network is missing")
disconnected = False
try:
docker("network", "disconnect", network, container)
disconnected = True
wait_health(container, "unhealthy")
for _ in range(20):
for path, schema in (("/api/v1/users/me", "User"),
("/health/ready", "Health")):
response = client.call("GET", path, actor=actor, schema=schema)
check(response["headers"]["X-Instance-Id"] == "api-2",
path + " was not served by healthy api2")
for number in range(10):
response = client.message(actor, chat, "unready check " + str(number))
check(response["headers"]["X-Instance-Id"] == "api-2",
"message was not accepted by healthy api2")
print("PASS: api1 unhealthy with frontend connection; 40 GETs and 10 POSTs served by api2")
finally:
if disconnected:
docker("network", "connect", "--alias", "api1", network, container)
wait_health(container, "healthy")
if __name__ == "__main__":
run(main)