forked from mtusi2026/mtusi2026lab5
254 lines
10 KiB
Python
254 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Real LB requests during each replica failure and sequential replacement."""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
from cluster import ROOT, compose, inspect, instance, instances, wait_lb
|
|
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure, check # noqa: E402
|
|
from resilience import history # noqa: E402
|
|
|
|
|
|
def percentile(values, fraction):
|
|
values = sorted(values)
|
|
return values[min(int((len(values) - 1) * fraction), len(values) - 1)] if values else None
|
|
|
|
|
|
class Traffic:
|
|
def __init__(self, base, actor, room, workers=6):
|
|
self.base, self.actor, self.room = base, actor, room
|
|
self.prefix = uuid4().hex
|
|
self.stop = threading.Event()
|
|
self.lock = threading.Lock()
|
|
self.events = []
|
|
self.executor = ThreadPoolExecutor(max_workers=workers)
|
|
self.workers = workers
|
|
|
|
def worker(self, worker):
|
|
client, iteration = Client(self.base, timeout=3), 0
|
|
while not self.stop.is_set():
|
|
started = time.monotonic()
|
|
write = iteration % 10 == 9
|
|
try:
|
|
if write:
|
|
key = f"{self.prefix}:{worker}:{iteration}"
|
|
response = client.message(self.actor, self.room, key, key=key)
|
|
else:
|
|
response = client.call(
|
|
"GET", "/api/v1/users/me", actor=self.actor, schema="User"
|
|
)
|
|
event = {
|
|
"at": time.monotonic(),
|
|
"ok": True,
|
|
"instance": response["headers"]["X-Instance-Id"],
|
|
"latency_ms": (time.monotonic() - started) * 1000,
|
|
"message": response["json"] if write else None,
|
|
}
|
|
iteration += 1
|
|
except Failure:
|
|
event = {
|
|
"at": time.monotonic(),
|
|
"ok": False,
|
|
"latency_ms": (time.monotonic() - started) * 1000,
|
|
}
|
|
with self.lock:
|
|
self.events.append(event)
|
|
|
|
def __enter__(self):
|
|
self.started = time.monotonic()
|
|
self.futures = [self.executor.submit(self.worker, i) for i in range(self.workers)]
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
self.stop.set()
|
|
self.executor.shutdown(wait=True)
|
|
for future in self.futures:
|
|
future.result()
|
|
self.finished = time.monotonic()
|
|
|
|
def report(self, action_start, action_end):
|
|
events = sorted(self.events, key=lambda row: row["at"])
|
|
ok = [row for row in events if row["ok"]]
|
|
after = [row for row in ok if row["at"] >= action_end]
|
|
check(bool(after), "No successful request after action")
|
|
elapsed = self.finished - self.started
|
|
latency = [row["latency_ms"] for row in ok]
|
|
return {
|
|
"duration_seconds": elapsed,
|
|
"workers": self.workers,
|
|
"requests": len(events),
|
|
"successes": len(ok),
|
|
"errors": len(events) - len(ok),
|
|
"successful_rps": len(ok) / elapsed,
|
|
"error_fraction": (len(events) - len(ok)) / len(events),
|
|
"first_success_after_action_seconds": after[0]["at"] - action_start,
|
|
"action_seconds": action_end - action_start,
|
|
"p50_ms": percentile(latency, 0.5),
|
|
"p95_ms": percentile(latency, 0.95),
|
|
"p99_ms": percentile(latency, 0.99),
|
|
"instances": sorted({row["instance"] for row in ok}),
|
|
"tail_instances": sorted({row["instance"] for row in after[-10:]}),
|
|
"confirmed_writes": sum(row["message"] is not None for row in ok),
|
|
}
|
|
|
|
def verify_messages(self, client, actor, room):
|
|
actual = history(client, actor, room)
|
|
by_id = {value["id"]: value for value in actual}
|
|
check(len(by_id) == len(actual), "History has duplicate IDs")
|
|
texts = [row["text"] for row in actual if row["text"].startswith(self.prefix)]
|
|
check(len(texts) == len(set(texts)), "An uncertain retry created a duplicate effect")
|
|
for event in self.events:
|
|
if event["ok"] and event["message"]:
|
|
check(
|
|
by_id.get(event["message"]["id"]) == event["message"],
|
|
"A confirmed message was lost or changed",
|
|
)
|
|
|
|
|
|
def public_test(target, base):
|
|
result = subprocess.run(
|
|
[
|
|
"make",
|
|
target,
|
|
"ACTION_SCRIPT=scripts/"
|
|
+ ("restart.sh" if target == "test-persistence" else "stop-one.sh"),
|
|
"BASE_URL=" + base,
|
|
],
|
|
cwd=ROOT,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=150,
|
|
)
|
|
return "\n".join(line.rstrip() for line in result.stdout.splitlines()) + "\n"
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--base-url", default="http://127.0.0.1:8085")
|
|
parser.add_argument(
|
|
"--output", type=Path, default=ROOT / f"evidence/cluster-{time.time_ns()}.json"
|
|
)
|
|
args = parser.parse_args()
|
|
if args.output.exists():
|
|
raise SystemExit("Use a new evidence filename")
|
|
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "127.0.0.1,localhost"
|
|
baseline = instances()
|
|
check(len(baseline) == 2, "This verification expects the default two API replicas")
|
|
check(not any(row["published_ports"] for row in baseline), "API must not publish host ports")
|
|
wait_lb(args.base_url, 2)
|
|
result = {"initial_instances": baseline, "failures": []}
|
|
client = Client(args.base_url, timeout=4)
|
|
alice, bob, eve, room = client.bootstrap()
|
|
key = uuid4().hex
|
|
|
|
def same_request(_):
|
|
response = Client(args.base_url).message(alice, room, "concurrent replay", key=key)
|
|
return response["json"], response["headers"]["X-Instance-Id"]
|
|
|
|
with ThreadPoolExecutor(max_workers=10) as pool:
|
|
responses = list(pool.map(same_request, range(30)))
|
|
check(
|
|
len({row[0]["id"] for row in responses}) == 1, "Cross-replica replay duplicated a message"
|
|
)
|
|
check(len({row[1] for row in responses}) == 2, "Concurrent replay did not cross real replicas")
|
|
result["concurrent_replays"] = 30
|
|
result["concurrent_effects"] = 1
|
|
try:
|
|
for selected in baseline:
|
|
selected = instance(selected["id"])
|
|
with Traffic(args.base_url, alice, room) as traffic:
|
|
time.sleep(1)
|
|
action_start = time.monotonic()
|
|
env = {**os.environ, "TARGET_INSTANCE_ID": selected["instance"]}
|
|
subprocess.run(
|
|
[str(ROOT / "scripts/stop-one.sh")],
|
|
cwd=ROOT,
|
|
env=env,
|
|
check=True,
|
|
capture_output=True,
|
|
timeout=10,
|
|
)
|
|
action_end = time.monotonic()
|
|
check(not inspect(selected["id"])["running"], "Chosen process was not stopped")
|
|
time.sleep(3)
|
|
report = traffic.report(action_start, action_end)
|
|
check(report["first_success_after_action_seconds"] <= 10, "Failover took too long")
|
|
check(selected["instance"] not in report["tail_instances"], "Stopped API still serves")
|
|
report["stopped"] = selected
|
|
traffic.verify_messages(client, bob, room)
|
|
result["failures"].append(report)
|
|
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
|
|
wait_lb(args.base_url, 2)
|
|
print("Verified failure and restoration of one API", flush=True)
|
|
with Traffic(args.base_url, alice, room) as traffic:
|
|
time.sleep(1)
|
|
action_start = time.monotonic()
|
|
replaced = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(ROOT / "scripts/rolling-update.py"),
|
|
"--base-url",
|
|
args.base_url,
|
|
],
|
|
cwd=ROOT,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=120,
|
|
)
|
|
action_end = time.monotonic()
|
|
time.sleep(3)
|
|
traffic.verify_messages(client, bob, room)
|
|
result["rolling"] = traffic.report(action_start, action_end)
|
|
result["rolling"]["replacements"] = json.loads(replaced.stdout)
|
|
result["rolling"]["confirmed_messages_preserved"] = True
|
|
print("Verified sequential replacement under traffic", flush=True)
|
|
(ROOT / "evidence/failover.txt").write_text(public_test("test-failover", args.base_url))
|
|
for value in compose("ps", "-aq", "api").split():
|
|
if not inspect(value)["running"]:
|
|
subprocess.run(["docker", "start", value], check=True, capture_output=True)
|
|
wait_lb(args.base_url, 2)
|
|
before = {
|
|
service: inspect(compose("ps", "-q", service).strip())
|
|
for service in ("postgres", "valkey", "s3", "broker", "lb")
|
|
}
|
|
(ROOT / "evidence/persistence.txt").write_text(
|
|
public_test("test-persistence", args.base_url)
|
|
)
|
|
after = {service: inspect(compose("ps", "-q", service).strip()) for service in before}
|
|
check(
|
|
all(before[s]["started_at"] != after[s]["started_at"] for s in before),
|
|
"Persistence hook did not restart all state/LB services",
|
|
)
|
|
result["persistence"] = {"before": before, "after": after, "same_token_public_test": True}
|
|
result["final_instances"] = instances()
|
|
logs = compose("logs", "--no-color", "api", "lb")
|
|
check(
|
|
not any(
|
|
a["token"] in logs or a["registration"]["password"] in logs
|
|
for a in (alice, bob, eve)
|
|
),
|
|
"Credential appeared in logs",
|
|
)
|
|
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
|
print("PASS: real replica failures, replay, rolling replacement and persistence")
|
|
finally:
|
|
for value in compose("ps", "-aq", "api").split():
|
|
if not inspect(value)["running"]:
|
|
subprocess.run(["docker", "start", value], check=True, capture_output=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|