Files
mtusi2026lab3/tests/load.py
T

165 lines
4.7 KiB
Python

#!/usr/bin/env python3
import subprocess
import sys
import threading
import time
from pathlib import Path
import requests
BASE = "http://localhost:8080"
LOGIN = "test"
PASSWORD = "test123"
TIMEOUT = 5
REPO_ROOT = Path(__file__).resolve().parents[1]
RECOVERY_BUDGET = 10.0
failures = []
def check(condition, message):
if condition:
print("PASS:", message)
return True
failures.append(message)
print("FAIL:", message)
return False
def login():
response = requests.post(
BASE + "/api/auth/login",
json={"login": LOGIN, "password": PASSWORD},
timeout=TIMEOUT,
)
response.raise_for_status()
return response.json()["token"]
def fetch(token):
response = requests.get(
BASE + "/api/messages",
headers={"Authorization": "Bearer " + token},
timeout=TIMEOUT,
)
response.raise_for_status()
return response.headers.get("X-Instance-Id", "")
def round_robin(token, count):
instances = []
lock = threading.Lock()
def worker():
try:
instance = fetch(token)
except requests.RequestException:
return
with lock:
instances.append(instance)
threads = [threading.Thread(target=worker) for _ in range(count)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
return instances
def compose(*args):
return subprocess.run(
["docker", "compose", *args],
cwd=REPO_ROOT,
capture_output=True,
text=True,
check=False,
)
def main():
token = login()
auth = {"Authorization": "Bearer " + token}
probe = requests.post(
BASE + "/api/messages",
headers=dict(auth, **{"Idempotency-Key": "load-" + str(time.time_ns())}),
json={"recipient": LOGIN, "body": "before failover"},
timeout=TIMEOUT,
)
check(probe.status_code == 201, "message written before failover (got %d)" % probe.status_code)
probe_id = probe.json().get("id")
instances = round_robin(token, 20)
check(len(set(instances)) >= 2, "requests reached at least two replicas: %s" % sorted(set(instances)))
print("\n=== Stopping api1 ===")
stop = compose("stop", "api1")
check(stop.returncode == 0, "docker compose stop api1 succeeded")
started = time.monotonic()
recovered_at = None
served_after = []
transient_errors = 0
while time.monotonic() - started < RECOVERY_BUDGET + 5:
try:
instance = fetch(token)
except requests.RequestException:
transient_errors += 1
time.sleep(0.2)
continue
if recovered_at is None:
recovered_at = time.monotonic() - started
served_after.append(instance)
if len(served_after) >= 5:
break
time.sleep(0.1)
check(
recovered_at is not None and recovered_at <= RECOVERY_BUDGET,
"reads recovered within %.0fs (%.2fs, %d transient errors)"
% (RECOVERY_BUDGET, recovered_at if recovered_at is not None else -1, transient_errors),
)
check(bool(served_after), "requests still served while api1 is down")
check(all(x == "api2" for x in served_after), "api1 left rotation, served by: %s" % sorted(set(served_after)))
write = requests.post(
BASE + "/api/messages",
headers=dict(auth, **{"Idempotency-Key": "load-fail-" + str(time.time_ns())}),
json={"recipient": LOGIN, "body": "written during failover"},
timeout=TIMEOUT,
)
check(write.status_code == 201, "write works while api1 is down (got %d)" % write.status_code)
print("\n=== Starting api1 back ===")
start = compose("start", "api1")
check(start.returncode == 0, "docker compose start api1 succeeded")
deadline = time.monotonic() + 30
back = set()
while time.monotonic() < deadline:
back = set(round_robin(token, 10))
if "api1" in back:
break
time.sleep(1)
check("api1" in back, "api1 is back in rotation: %s" % sorted(back))
history = requests.get(BASE + "/api/messages", headers=auth, timeout=TIMEOUT)
check(history.status_code == 200, "history readable after failover (got %d)" % history.status_code)
ids = [m.get("id") for m in history.json()]
check(probe_id in ids, "message written before failover survived")
check(len(ids) == len(set(ids)), "no duplicate messages in history")
if failures:
print("\n%d check(s) failed" % len(failures))
return 1
print("\nFailover scenario passed")
return 0
if __name__ == "__main__":
try:
sys.exit(main())
except requests.RequestException as exc:
print("request failed:", exc)
sys.exit(1)