"""Stop each real API process under traffic; retain confirmed writes and source aliases.""" import json import os import subprocess import sys import threading import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure def compose(*args): return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True) def aliases(): result = {} for cid in compose("ps", "-q", "api").split(): alias = subprocess.check_output( [ "docker", "exec", cid, "python", "-c", "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])", ], text=True, ).strip() result[cid] = alias return result def main(): compose("up", "-d", "--no-recreate", "--wait") subprocess.run([sys.executable, "scripts/wait_replicas.py", "2"], cwd=ROOT, check=True) c = Client("http://127.0.0.1:8083") actor, _, _, chat = c.bootstrap() initial = aliases() runs = [] for target_container, target_alias in initial.items(): successful, errors = [], [] stop = threading.Event() stopped_at = None def traffic(): while not stop.is_set(): try: response = c.message(actor, chat, "failover-proof") successful.append( { "at": time.monotonic(), "id": response["json"]["id"], "instance": response["headers"]["X-Instance-Id"], } ) except Failure: errors.append(time.monotonic()) started = time.monotonic() thread = threading.Thread(target=traffic) thread.start() time.sleep(1) stopped_at = time.monotonic() subprocess.run( ["scripts/stop-one.sh"], cwd=ROOT, env={**os.environ, "TARGET_INSTANCE_ID": target_alias}, check=True, ) time.sleep(4) stop.set() thread.join() elapsed = time.monotonic() - started after = [x for x in successful if x["at"] > stopped_at] assert after and all(x["instance"] != target_alias for x in after[-5:]) assert min(x["at"] for x in after) - stopped_at <= 10 state = json.loads( subprocess.check_output(["docker", "inspect", target_container], text=True) )[0]["State"] assert not state["Running"] runs.append( { "target_container": target_container, "target_alias": target_alias, "successful_writes": len(successful), "failed_attempts": len(errors), "successful_rps": len(successful) / elapsed, "recovery_seconds": min(x["at"] for x in after) - stopped_at, "observed_after": sorted({x["instance"] for x in after[-5:]}), "target_running_after_hook": state["Running"], } ) subprocess.run(["docker", "start", target_container], check=True, stdout=subprocess.DEVNULL) subprocess.run([sys.executable, "scripts/wait_replicas.py", "2"], cwd=ROOT, check=True) # Verify every acknowledged write, without assuming failed requests did not commit. items, cursor = [], None while True: path = f"/api/v1/chats/{chat['id']}/messages?limit=100" if cursor: path += "&cursor=" + cursor page = c.call("GET", path, actor=actor, schema="MessagePage")["json"] items.extend(page["items"]) cursor = page["next_cursor"] if cursor is None: break ids = {x["id"] for x in items} assert all(x["id"] in ids for x in successful) runs[-1]["all_confirmed_present"] = True report = {"real_instances_before": initial, "runs": runs, "pass": True} (ROOT / "evidence/replica-failures.json").write_text(json.dumps(report, indent=2) + "\n") print(json.dumps(report, indent=2)) if __name__ == "__main__": main()