Files
mtusi2026lab5/scripts/verify_cluster.py
T

254 lines
10 KiB
Python

#!/usr/bin/env python3
"""Real LB requests during each replica failure and sequential replacement."""
import argparse
import json
import os
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from uuid import uuid4
from cluster import ROOT, compose, inspect, instance, instances, wait_lb
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check # noqa: E402
from resilience import history # noqa: E402
def percentile(values, fraction):
values = sorted(values)
return values[min(int((len(values) - 1) * fraction), len(values) - 1)] if values else None
class Traffic:
def __init__(self, base, actor, room, workers=6):
self.base, self.actor, self.room = base, actor, room
self.prefix = uuid4().hex
self.stop = threading.Event()
self.lock = threading.Lock()
self.events = []
self.executor = ThreadPoolExecutor(max_workers=workers)
self.workers = workers
def worker(self, worker):
client, iteration = Client(self.base, timeout=3), 0
while not self.stop.is_set():
started = time.monotonic()
write = iteration % 10 == 9
try:
if write:
key = f"{self.prefix}:{worker}:{iteration}"
response = client.message(self.actor, self.room, key, key=key)
else:
response = client.call(
"GET", "/api/v1/users/me", actor=self.actor, schema="User"
)
event = {
"at": time.monotonic(),
"ok": True,
"instance": response["headers"]["X-Instance-Id"],
"latency_ms": (time.monotonic() - started) * 1000,
"message": response["json"] if write else None,
}
iteration += 1
except Failure:
event = {
"at": time.monotonic(),
"ok": False,
"latency_ms": (time.monotonic() - started) * 1000,
}
with self.lock:
self.events.append(event)
def __enter__(self):
self.started = time.monotonic()
self.futures = [self.executor.submit(self.worker, i) for i in range(self.workers)]
return self
def __exit__(self, *args):
self.stop.set()
self.executor.shutdown(wait=True)
for future in self.futures:
future.result()
self.finished = time.monotonic()
def report(self, action_start, action_end):
events = sorted(self.events, key=lambda row: row["at"])
ok = [row for row in events if row["ok"]]
after = [row for row in ok if row["at"] >= action_end]
check(bool(after), "No successful request after action")
elapsed = self.finished - self.started
latency = [row["latency_ms"] for row in ok]
return {
"duration_seconds": elapsed,
"workers": self.workers,
"requests": len(events),
"successes": len(ok),
"errors": len(events) - len(ok),
"successful_rps": len(ok) / elapsed,
"error_fraction": (len(events) - len(ok)) / len(events),
"first_success_after_action_seconds": after[0]["at"] - action_start,
"action_seconds": action_end - action_start,
"p50_ms": percentile(latency, 0.5),
"p95_ms": percentile(latency, 0.95),
"p99_ms": percentile(latency, 0.99),
"instances": sorted({row["instance"] for row in ok}),
"tail_instances": sorted({row["instance"] for row in after[-10:]}),
"confirmed_writes": sum(row["message"] is not None for row in ok),
}
def verify_messages(self, client, actor, room):
actual = history(client, actor, room)
by_id = {value["id"]: value for value in actual}
check(len(by_id) == len(actual), "History has duplicate IDs")
texts = [row["text"] for row in actual if row["text"].startswith(self.prefix)]
check(len(texts) == len(set(texts)), "An uncertain retry created a duplicate effect")
for event in self.events:
if event["ok"] and event["message"]:
check(
by_id.get(event["message"]["id"]) == event["message"],
"A confirmed message was lost or changed",
)
def public_test(target, base):
result = subprocess.run(
[
"make",
target,
"ACTION_SCRIPT=scripts/"
+ ("restart.sh" if target == "test-persistence" else "stop-one.sh"),
"BASE_URL=" + base,
],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
timeout=150,
)
return "\n".join(line.rstrip() for line in result.stdout.splitlines()) + "\n"
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8085")
parser.add_argument(
"--output", type=Path, default=ROOT / f"evidence/cluster-{time.time_ns()}.json"
)
args = parser.parse_args()
if args.output.exists():
raise SystemExit("Use a new evidence filename")
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "127.0.0.1,localhost"
baseline = instances()
check(len(baseline) == 2, "This verification expects the default two API replicas")
check(not any(row["published_ports"] for row in baseline), "API must not publish host ports")
wait_lb(args.base_url, 2)
result = {"initial_instances": baseline, "failures": []}
client = Client(args.base_url, timeout=4)
alice, bob, eve, room = client.bootstrap()
key = uuid4().hex
def same_request(_):
response = Client(args.base_url).message(alice, room, "concurrent replay", key=key)
return response["json"], response["headers"]["X-Instance-Id"]
with ThreadPoolExecutor(max_workers=10) as pool:
responses = list(pool.map(same_request, range(30)))
check(
len({row[0]["id"] for row in responses}) == 1, "Cross-replica replay duplicated a message"
)
check(len({row[1] for row in responses}) == 2, "Concurrent replay did not cross real replicas")
result["concurrent_replays"] = 30
result["concurrent_effects"] = 1
try:
for selected in baseline:
selected = instance(selected["id"])
with Traffic(args.base_url, alice, room) as traffic:
time.sleep(1)
action_start = time.monotonic()
env = {**os.environ, "TARGET_INSTANCE_ID": selected["instance"]}
subprocess.run(
[str(ROOT / "scripts/stop-one.sh")],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
timeout=10,
)
action_end = time.monotonic()
check(not inspect(selected["id"])["running"], "Chosen process was not stopped")
time.sleep(3)
report = traffic.report(action_start, action_end)
check(report["first_success_after_action_seconds"] <= 10, "Failover took too long")
check(selected["instance"] not in report["tail_instances"], "Stopped API still serves")
report["stopped"] = selected
traffic.verify_messages(client, bob, room)
result["failures"].append(report)
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
wait_lb(args.base_url, 2)
print("Verified failure and restoration of one API", flush=True)
with Traffic(args.base_url, alice, room) as traffic:
time.sleep(1)
action_start = time.monotonic()
replaced = subprocess.run(
[
sys.executable,
str(ROOT / "scripts/rolling-update.py"),
"--base-url",
args.base_url,
],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
timeout=120,
)
action_end = time.monotonic()
time.sleep(3)
traffic.verify_messages(client, bob, room)
result["rolling"] = traffic.report(action_start, action_end)
result["rolling"]["replacements"] = json.loads(replaced.stdout)
result["rolling"]["confirmed_messages_preserved"] = True
print("Verified sequential replacement under traffic", flush=True)
(ROOT / "evidence/failover.txt").write_text(public_test("test-failover", args.base_url))
for value in compose("ps", "-aq", "api").split():
if not inspect(value)["running"]:
subprocess.run(["docker", "start", value], check=True, capture_output=True)
wait_lb(args.base_url, 2)
before = {
service: inspect(compose("ps", "-q", service).strip())
for service in ("postgres", "valkey", "s3", "broker", "lb")
}
(ROOT / "evidence/persistence.txt").write_text(
public_test("test-persistence", args.base_url)
)
after = {service: inspect(compose("ps", "-q", service).strip()) for service in before}
check(
all(before[s]["started_at"] != after[s]["started_at"] for s in before),
"Persistence hook did not restart all state/LB services",
)
result["persistence"] = {"before": before, "after": after, "same_token_public_test": True}
result["final_instances"] = instances()
logs = compose("logs", "--no-color", "api", "lb")
check(
not any(
a["token"] in logs or a["registration"]["password"] in logs
for a in (alice, bob, eve)
),
"Credential appeared in logs",
)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print("PASS: real replica failures, replay, rolling replacement and persistence")
finally:
for value in compose("ps", "-aq", "api").split():
if not inspect(value)["running"]:
subprocess.run(["docker", "start", value], check=True, capture_output=True)
if __name__ == "__main__":
main()