forked from mtusi2026/mtusi2026lab3
165 lines
4.7 KiB
Python
165 lines
4.7 KiB
Python
#!/usr/bin/env python3
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
|
|
BASE = "http://localhost:8080"
|
|
LOGIN = "test"
|
|
PASSWORD = "test123"
|
|
TIMEOUT = 5
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
RECOVERY_BUDGET = 10.0
|
|
|
|
failures = []
|
|
|
|
|
|
def check(condition, message):
|
|
if condition:
|
|
print("PASS:", message)
|
|
return True
|
|
failures.append(message)
|
|
print("FAIL:", message)
|
|
return False
|
|
|
|
|
|
def login():
|
|
response = requests.post(
|
|
BASE + "/api/auth/login",
|
|
json={"login": LOGIN, "password": PASSWORD},
|
|
timeout=TIMEOUT,
|
|
)
|
|
response.raise_for_status()
|
|
return response.json()["token"]
|
|
|
|
|
|
def fetch(token):
|
|
response = requests.get(
|
|
BASE + "/api/messages",
|
|
headers={"Authorization": "Bearer " + token},
|
|
timeout=TIMEOUT,
|
|
)
|
|
response.raise_for_status()
|
|
return response.headers.get("X-Instance-Id", "")
|
|
|
|
|
|
def round_robin(token, count):
|
|
instances = []
|
|
lock = threading.Lock()
|
|
|
|
def worker():
|
|
try:
|
|
instance = fetch(token)
|
|
except requests.RequestException:
|
|
return
|
|
with lock:
|
|
instances.append(instance)
|
|
|
|
threads = [threading.Thread(target=worker) for _ in range(count)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join()
|
|
return instances
|
|
|
|
|
|
def compose(*args):
|
|
return subprocess.run(
|
|
["docker", "compose", *args],
|
|
cwd=REPO_ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
|
|
|
|
def main():
|
|
token = login()
|
|
auth = {"Authorization": "Bearer " + token}
|
|
|
|
probe = requests.post(
|
|
BASE + "/api/messages",
|
|
headers=dict(auth, **{"Idempotency-Key": "load-" + str(time.time_ns())}),
|
|
json={"recipient": LOGIN, "body": "before failover"},
|
|
timeout=TIMEOUT,
|
|
)
|
|
check(probe.status_code == 201, "message written before failover (got %d)" % probe.status_code)
|
|
probe_id = probe.json().get("id")
|
|
|
|
instances = round_robin(token, 20)
|
|
check(len(set(instances)) >= 2, "requests reached at least two replicas: %s" % sorted(set(instances)))
|
|
|
|
print("\n=== Stopping api1 ===")
|
|
stop = compose("stop", "api1")
|
|
check(stop.returncode == 0, "docker compose stop api1 succeeded")
|
|
started = time.monotonic()
|
|
|
|
recovered_at = None
|
|
served_after = []
|
|
transient_errors = 0
|
|
while time.monotonic() - started < RECOVERY_BUDGET + 5:
|
|
try:
|
|
instance = fetch(token)
|
|
except requests.RequestException:
|
|
transient_errors += 1
|
|
time.sleep(0.2)
|
|
continue
|
|
if recovered_at is None:
|
|
recovered_at = time.monotonic() - started
|
|
served_after.append(instance)
|
|
if len(served_after) >= 5:
|
|
break
|
|
time.sleep(0.1)
|
|
|
|
check(
|
|
recovered_at is not None and recovered_at <= RECOVERY_BUDGET,
|
|
"reads recovered within %.0fs (%.2fs, %d transient errors)"
|
|
% (RECOVERY_BUDGET, recovered_at if recovered_at is not None else -1, transient_errors),
|
|
)
|
|
check(bool(served_after), "requests still served while api1 is down")
|
|
check(all(x == "api2" for x in served_after), "api1 left rotation, served by: %s" % sorted(set(served_after)))
|
|
|
|
write = requests.post(
|
|
BASE + "/api/messages",
|
|
headers=dict(auth, **{"Idempotency-Key": "load-fail-" + str(time.time_ns())}),
|
|
json={"recipient": LOGIN, "body": "written during failover"},
|
|
timeout=TIMEOUT,
|
|
)
|
|
check(write.status_code == 201, "write works while api1 is down (got %d)" % write.status_code)
|
|
|
|
print("\n=== Starting api1 back ===")
|
|
start = compose("start", "api1")
|
|
check(start.returncode == 0, "docker compose start api1 succeeded")
|
|
|
|
deadline = time.monotonic() + 30
|
|
back = set()
|
|
while time.monotonic() < deadline:
|
|
back = set(round_robin(token, 10))
|
|
if "api1" in back:
|
|
break
|
|
time.sleep(1)
|
|
check("api1" in back, "api1 is back in rotation: %s" % sorted(back))
|
|
|
|
history = requests.get(BASE + "/api/messages", headers=auth, timeout=TIMEOUT)
|
|
check(history.status_code == 200, "history readable after failover (got %d)" % history.status_code)
|
|
ids = [m.get("id") for m in history.json()]
|
|
check(probe_id in ids, "message written before failover survived")
|
|
check(len(ids) == len(set(ids)), "no duplicate messages in history")
|
|
|
|
if failures:
|
|
print("\n%d check(s) failed" % len(failures))
|
|
return 1
|
|
print("\nFailover scenario passed")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
sys.exit(main())
|
|
except requests.RequestException as exc:
|
|
print("request failed:", exc)
|
|
sys.exit(1)
|