#!/usr/bin/env python3 """Real LB requests during each replica failure and sequential replacement.""" import argparse import json import os import subprocess import sys import threading import time from concurrent.futures import ThreadPoolExecutor from pathlib import Path from uuid import uuid4 from cluster import ROOT, compose, inspect, instance, instances, wait_lb sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure, check # noqa: E402 from resilience import history # noqa: E402 def percentile(values, fraction): values = sorted(values) return values[min(int((len(values) - 1) * fraction), len(values) - 1)] if values else None class Traffic: def __init__(self, base, actor, room, workers=6): self.base, self.actor, self.room = base, actor, room self.prefix = uuid4().hex self.stop = threading.Event() self.lock = threading.Lock() self.events = [] self.executor = ThreadPoolExecutor(max_workers=workers) self.workers = workers def worker(self, worker): client, iteration = Client(self.base, timeout=3), 0 while not self.stop.is_set(): started = time.monotonic() write = iteration % 10 == 9 try: if write: key = f"{self.prefix}:{worker}:{iteration}" response = client.message(self.actor, self.room, key, key=key) else: response = client.call( "GET", "/api/v1/users/me", actor=self.actor, schema="User" ) event = { "at": time.monotonic(), "ok": True, "instance": response["headers"]["X-Instance-Id"], "latency_ms": (time.monotonic() - started) * 1000, "message": response["json"] if write else None, } iteration += 1 except Failure: event = { "at": time.monotonic(), "ok": False, "latency_ms": (time.monotonic() - started) * 1000, } with self.lock: self.events.append(event) def __enter__(self): self.started = time.monotonic() self.futures = [self.executor.submit(self.worker, i) for i in range(self.workers)] return self def __exit__(self, *args): self.stop.set() self.executor.shutdown(wait=True) for future in self.futures: future.result() self.finished = time.monotonic() def report(self, action_start, action_end): events = sorted(self.events, key=lambda row: row["at"]) ok = [row for row in events if row["ok"]] after = [row for row in ok if row["at"] >= action_end] check(bool(after), "No successful request after action") elapsed = self.finished - self.started latency = [row["latency_ms"] for row in ok] return { "duration_seconds": elapsed, "workers": self.workers, "requests": len(events), "successes": len(ok), "errors": len(events) - len(ok), "successful_rps": len(ok) / elapsed, "error_fraction": (len(events) - len(ok)) / len(events), "first_success_after_action_seconds": after[0]["at"] - action_start, "action_seconds": action_end - action_start, "p50_ms": percentile(latency, 0.5), "p95_ms": percentile(latency, 0.95), "p99_ms": percentile(latency, 0.99), "instances": sorted({row["instance"] for row in ok}), "tail_instances": sorted({row["instance"] for row in after[-10:]}), "confirmed_writes": sum(row["message"] is not None for row in ok), } def verify_messages(self, client, actor, room): actual = history(client, actor, room) by_id = {value["id"]: value for value in actual} check(len(by_id) == len(actual), "History has duplicate IDs") texts = [row["text"] for row in actual if row["text"].startswith(self.prefix)] check(len(texts) == len(set(texts)), "An uncertain retry created a duplicate effect") for event in self.events: if event["ok"] and event["message"]: check( by_id.get(event["message"]["id"]) == event["message"], "A confirmed message was lost or changed", ) def public_test(target, base): result = subprocess.run( [ "make", target, "ACTION_SCRIPT=scripts/" + ("restart.sh" if target == "test-persistence" else "stop-one.sh"), "BASE_URL=" + base, ], cwd=ROOT, check=True, capture_output=True, text=True, timeout=150, ) return "\n".join(line.rstrip() for line in result.stdout.splitlines()) + "\n" def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-url", default="http://127.0.0.1:8084") parser.add_argument( "--output", type=Path, default=ROOT / f"evidence/cluster-{time.time_ns()}.json" ) args = parser.parse_args() if args.output.exists(): raise SystemExit("Use a new evidence filename") os.environ["NO_PROXY"] = os.environ["no_proxy"] = "127.0.0.1,localhost" baseline = instances() check(len(baseline) == 2, "This verification expects the default two API replicas") check(not any(row["published_ports"] for row in baseline), "API must not publish host ports") wait_lb(args.base_url, 2) result = {"initial_instances": baseline, "failures": []} client = Client(args.base_url, timeout=4) alice, bob, eve, room = client.bootstrap() key = uuid4().hex def same_request(_): response = Client(args.base_url).message(alice, room, "concurrent replay", key=key) return response["json"], response["headers"]["X-Instance-Id"] with ThreadPoolExecutor(max_workers=10) as pool: responses = list(pool.map(same_request, range(30))) check( len({row[0]["id"] for row in responses}) == 1, "Cross-replica replay duplicated a message" ) check(len({row[1] for row in responses}) == 2, "Concurrent replay did not cross real replicas") result["concurrent_replays"] = 30 result["concurrent_effects"] = 1 try: for selected in baseline: selected = instance(selected["id"]) with Traffic(args.base_url, alice, room) as traffic: time.sleep(1) action_start = time.monotonic() env = {**os.environ, "TARGET_INSTANCE_ID": selected["instance"]} subprocess.run( [str(ROOT / "scripts/stop-one.sh")], cwd=ROOT, env=env, check=True, capture_output=True, timeout=10, ) action_end = time.monotonic() check(not inspect(selected["id"])["running"], "Chosen process was not stopped") time.sleep(3) report = traffic.report(action_start, action_end) check(report["first_success_after_action_seconds"] <= 10, "Failover took too long") check(selected["instance"] not in report["tail_instances"], "Stopped API still serves") report["stopped"] = selected traffic.verify_messages(client, bob, room) result["failures"].append(report) subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True) wait_lb(args.base_url, 2) print("Verified failure and restoration of one API", flush=True) with Traffic(args.base_url, alice, room) as traffic: time.sleep(1) action_start = time.monotonic() replaced = subprocess.run( [ sys.executable, str(ROOT / "scripts/rolling-update.py"), "--base-url", args.base_url, ], cwd=ROOT, check=True, capture_output=True, text=True, timeout=120, ) action_end = time.monotonic() time.sleep(3) traffic.verify_messages(client, bob, room) result["rolling"] = traffic.report(action_start, action_end) result["rolling"]["replacements"] = json.loads(replaced.stdout) result["rolling"]["confirmed_messages_preserved"] = True print("Verified sequential replacement under traffic", flush=True) (ROOT / "evidence/failover.txt").write_text(public_test("test-failover", args.base_url)) for value in compose("ps", "-aq", "api").split(): if not inspect(value)["running"]: subprocess.run(["docker", "start", value], check=True, capture_output=True) wait_lb(args.base_url, 2) before = { service: inspect(compose("ps", "-q", service).strip()) for service in ("postgres", "valkey", "s3", "lb") } (ROOT / "evidence/persistence.txt").write_text( public_test("test-persistence", args.base_url) ) after = {service: inspect(compose("ps", "-q", service).strip()) for service in before} check( all(before[s]["started_at"] != after[s]["started_at"] for s in before), "Persistence hook did not restart all state/LB services", ) result["persistence"] = {"before": before, "after": after, "same_token_public_test": True} result["final_instances"] = instances() logs = compose("logs", "--no-color", "api", "lb") check( not any( a["token"] in logs or a["registration"]["password"] in logs for a in (alice, bob, eve) ), "Credential appeared in logs", ) args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") print("PASS: real replica failures, replay, rolling replacement and persistence") finally: for value in compose("ps", "-aq", "api").split(): if not inspect(value)["running"]: subprocess.run(["docker", "start", value], check=True, capture_output=True) if __name__ == "__main__": main()