#!/usr/bin/env python3 """Check failover of a named Compose API replica without changing public tests.""" import argparse import os from pathlib import Path import subprocess import sys import time import uuid ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure, check # noqa: E402 def main(): parser = argparse.ArgumentParser() parser.add_argument("--base-url", default="http://127.0.0.1:8080") parser.add_argument("--target-index", type=int, choices=(1, 2), required=True) args = parser.parse_args() ids = subprocess.run( ["docker", "compose", "ps", "--all", "-q", "api"], cwd=ROOT, capture_output=True, text=True, check=True, ).stdout.splitlines() check(len(ids) == 2, "targeted failover requires two API replicas") target = subprocess.run( ["docker", "inspect", "--format", "{{.Config.Hostname}}", ids[args.target_index - 1]], cwd=ROOT, capture_output=True, text=True, check=True, ).stdout.strip() expected = { subprocess.run( ["docker", "inspect", "--format", "{{.Config.Hostname}}", item], cwd=ROOT, capture_output=True, text=True, check=True, ).stdout.strip() for item in ids } client = Client(args.base_url, timeout=10) alice, bob, _, chat = client.bootstrap() before = client.message(alice, chat, "Before targeted failover")["json"] observed = { client.call("GET", "/api/v1/users/me", actor=alice, schema="User")[ "headers" ]["X-Instance-Id"] for _ in range(40) } check(observed == expected, "both replicas must answer before failure") client.timeout = 2 hook = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh") env = dict(os.environ, TARGET_INSTANCE_ID=target) started = time.monotonic() result = subprocess.run( [str(hook)], cwd=ROOT, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10, check=False, ) check(result.returncode == 0, "stop-one hook failed") key = uuid.uuid4().hex text = "Targeted failover " + key failures = 0 while True: try: written = client.message(alice, chat, text, key=key) break except Failure: failures += 1 check(time.monotonic() - started <= 10, "write recovery exceeded 10 seconds") time.sleep(0.1) recovery = time.monotonic() - started check(recovery <= 10, "write recovery exceeded 10 seconds") check(written["headers"]["X-Instance-Id"] != target, "stopped replica answered the write") reads = 0 first_read = None while reads < 10: try: response = client.call("GET", "/api/v1/users/me", actor=alice, schema="User") except Failure: failures += 1 check(time.monotonic() - started <= 10, "read recovery exceeded 10 seconds") continue if first_read is None: first_read = time.monotonic() - started check(first_read <= 10, "read recovery exceeded 10 seconds") check(response["headers"]["X-Instance-Id"] != target, "stopped replica still serves traffic") reads += 1 messages = client.call( "GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", actor=bob, schema="MessagePage", )["json"]["items"] check(before in messages, "confirmed pre-failure message disappeared") check(sum(message["text"] == text for message in messages) == 1, "uncertain retry created a duplicate") print(f"PASS: replica {args.target_index} stopped; write_recovery={recovery:.3f}s; " f"read_recovery={first_read:.3f}s; transient_failures={failures}") if __name__ == "__main__": main()