forked from mtusi2026/mtusi2026lab3
61 lines
2.3 KiB
Python
61 lines
2.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Check that HAProxy removes a live API with failed database readiness."""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
|
|
from client import Client, Failure
|
|
|
|
|
|
PROJECT = "mtusi2026lab3-yurtaev"
|
|
TARGET = PROJECT + "-api-1"
|
|
NETWORK = PROJECT + "_data"
|
|
|
|
|
|
def docker(*args):
|
|
return subprocess.check_output(["docker", *args], text=True).strip()
|
|
|
|
|
|
def main():
|
|
client = Client("http://127.0.0.1:8080", timeout=2)
|
|
alice, _, _, chat = client.bootstrap()
|
|
target_id = docker("exec", TARGET, "python", "-c",
|
|
"import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])")
|
|
before = {client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"]
|
|
for _ in range(40)}
|
|
assert len(before) == 2 and target_id in before, (before, target_id)
|
|
start = time.monotonic()
|
|
docker("network", "disconnect", NETWORK, TARGET)
|
|
try:
|
|
consecutive = 0
|
|
errors = 0
|
|
detected_at = None
|
|
while time.monotonic() - start < 10:
|
|
try:
|
|
response = client.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
|
assert response["headers"]["X-Instance-Id"] != target_id
|
|
consecutive += 1
|
|
except Failure:
|
|
errors += 1
|
|
consecutive = 0
|
|
if consecutive >= 20:
|
|
detected_at = time.monotonic() - start
|
|
break
|
|
time.sleep(0.05)
|
|
assert detected_at is not None, "HAProxy did not exclude unready replica"
|
|
messages = [client.message(alice, chat, "readiness probe " + str(n))["json"] for n in range(10)]
|
|
assert len({message["id"] for message in messages}) == 10
|
|
report = {"target_instance_id": target_id, "observed_before": sorted(before),
|
|
"excluded_within_seconds": round(detected_at, 3), "transient_errors": errors,
|
|
"successful_writes_after_exclusion": len(messages)}
|
|
Path("evidence/readiness_failover.json").write_text(json.dumps(report, indent=2) + "\n")
|
|
print(json.dumps(report, indent=2))
|
|
finally:
|
|
docker("network", "connect", NETWORK, TARGET)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|