#!/usr/bin/env python3 import subprocess import sys import threading import time from pathlib import Path import requests BASE = "http://localhost:8080" LOGIN = "test" PASSWORD = "test123" TIMEOUT = 5 REPO_ROOT = Path(__file__).resolve().parents[1] RECOVERY_BUDGET = 10.0 failures = [] def check(condition, message): if condition: print("PASS:", message) return True failures.append(message) print("FAIL:", message) return False def login(): response = requests.post( BASE + "/api/auth/login", json={"login": LOGIN, "password": PASSWORD}, timeout=TIMEOUT, ) response.raise_for_status() return response.json()["token"] def fetch(token): response = requests.get( BASE + "/api/messages", headers={"Authorization": "Bearer " + token}, timeout=TIMEOUT, ) response.raise_for_status() return response.headers.get("X-Instance-Id", "") def round_robin(token, count): instances = [] lock = threading.Lock() def worker(): try: instance = fetch(token) except requests.RequestException: return with lock: instances.append(instance) threads = [threading.Thread(target=worker) for _ in range(count)] for thread in threads: thread.start() for thread in threads: thread.join() return instances def compose(*args): return subprocess.run( ["docker", "compose", *args], cwd=REPO_ROOT, capture_output=True, text=True, check=False, ) def main(): token = login() auth = {"Authorization": "Bearer " + token} probe = requests.post( BASE + "/api/messages", headers=dict(auth, **{"Idempotency-Key": "load-" + str(time.time_ns())}), json={"recipient": LOGIN, "body": "before failover"}, timeout=TIMEOUT, ) check(probe.status_code == 201, "message written before failover (got %d)" % probe.status_code) probe_id = probe.json().get("id") instances = round_robin(token, 20) check(len(set(instances)) >= 2, "requests reached at least two replicas: %s" % sorted(set(instances))) print("\n=== Stopping api1 ===") stop = compose("stop", "api1") check(stop.returncode == 0, "docker compose stop api1 succeeded") started = time.monotonic() recovered_at = None served_after = [] transient_errors = 0 while time.monotonic() - started < RECOVERY_BUDGET + 5: try: instance = fetch(token) except requests.RequestException: transient_errors += 1 time.sleep(0.2) continue if recovered_at is None: recovered_at = time.monotonic() - started served_after.append(instance) if len(served_after) >= 5: break time.sleep(0.1) check( recovered_at is not None and recovered_at <= RECOVERY_BUDGET, "reads recovered within %.0fs (%.2fs, %d transient errors)" % (RECOVERY_BUDGET, recovered_at if recovered_at is not None else -1, transient_errors), ) check(bool(served_after), "requests still served while api1 is down") check(all(x == "api2" for x in served_after), "api1 left rotation, served by: %s" % sorted(set(served_after))) write = requests.post( BASE + "/api/messages", headers=dict(auth, **{"Idempotency-Key": "load-fail-" + str(time.time_ns())}), json={"recipient": LOGIN, "body": "written during failover"}, timeout=TIMEOUT, ) check(write.status_code == 201, "write works while api1 is down (got %d)" % write.status_code) print("\n=== Starting api1 back ===") start = compose("start", "api1") check(start.returncode == 0, "docker compose start api1 succeeded") deadline = time.monotonic() + 30 back = set() while time.monotonic() < deadline: back = set(round_robin(token, 10)) if "api1" in back: break time.sleep(1) check("api1" in back, "api1 is back in rotation: %s" % sorted(back)) history = requests.get(BASE + "/api/messages", headers=auth, timeout=TIMEOUT) check(history.status_code == 200, "history readable after failover (got %d)" % history.status_code) ids = [m.get("id") for m in history.json()] check(probe_id in ids, "message written before failover survived") check(len(ids) == len(set(ids)), "no duplicate messages in history") if failures: print("\n%d check(s) failed" % len(failures)) return 1 print("\nFailover scenario passed") return 0 if __name__ == "__main__": try: sys.exit(main()) except requests.RequestException as exc: print("request failed:", exc) sys.exit(1)