forked from mtusi2026/mtusi2026lab3
test: cover persistence failover and graceful draining
Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
This commit is contained in:
+332
@@ -0,0 +1,332 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab 3 grade 4: measured replica failover or sequential replacement."""
|
||||
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
from resilience import history # noqa: E402
|
||||
|
||||
def compose(*args, timeout=60):
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "-f", "compose.yaml", "-f", "configs/two.yaml", *args],
|
||||
cwd=ROOT, capture_output=True,
|
||||
text=True, timeout=timeout, check=False,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
raise Failure(f"docker compose {args[0]} failed: {type(exc).__name__}") from None
|
||||
check(result.returncode == 0, f"docker compose {args[0]} failed; inspect Docker logs")
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def docker(*args, timeout=60):
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", *args], cwd=ROOT, capture_output=True,
|
||||
text=True, timeout=timeout, check=False,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
raise Failure(f"docker {args[0]} failed: {type(exc).__name__}") from None
|
||||
check(result.returncode == 0, f"docker {args[0]} failed; inspect Docker logs")
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def inspect(container_id, field):
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "inspect", "--format", field, container_id],
|
||||
cwd=ROOT, capture_output=True, text=True, timeout=10, check=False,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
raise Failure(f"docker inspect failed: {type(exc).__name__}") from None
|
||||
check(result.returncode == 0, "docker inspect failed")
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def containers(service):
|
||||
ids = compose("ps", "--all", "-q", service).splitlines()
|
||||
check(bool(ids), f"{service} container is missing")
|
||||
return ids
|
||||
|
||||
|
||||
def container(service):
|
||||
ids = containers(service)
|
||||
check(len(ids) == 1, f"expected one {service} container")
|
||||
return ids[0]
|
||||
|
||||
|
||||
def instance(container_id):
|
||||
return inspect(container_id, "{{.Config.Hostname}}")
|
||||
|
||||
|
||||
def healthy(container_id, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if inspect(container_id, "{{.State.Health.Status}}") == "healthy":
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise Failure(f"API container did not become healthy in {timeout} seconds")
|
||||
|
||||
|
||||
def both_observed(client, actor, expected, timeout=30):
|
||||
seen = set()
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
response = client.call("GET", "/api/v1/users/me", actor=actor,
|
||||
schema="User")
|
||||
seen.add(response["headers"]["X-Instance-Id"])
|
||||
except Failure:
|
||||
pass
|
||||
if seen == expected:
|
||||
return
|
||||
time.sleep(0.1)
|
||||
raise Failure(f"both replicas were not observed; saw {sorted(seen)}")
|
||||
|
||||
|
||||
class Traffic:
|
||||
def __init__(self, base_url, actor, chat, concurrency):
|
||||
self.base_url = base_url
|
||||
self.actor = actor
|
||||
self.chat = chat
|
||||
self.concurrency = concurrency
|
||||
self.stop = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.events = []
|
||||
self.confirmed = set()
|
||||
self.attempted = set()
|
||||
self.pool = ThreadPoolExecutor(max_workers=concurrency)
|
||||
|
||||
def start(self):
|
||||
self.futures = [self.pool.submit(self.worker, i)
|
||||
for i in range(self.concurrency)]
|
||||
|
||||
def worker(self, worker_id):
|
||||
client = Client(self.base_url, timeout=2)
|
||||
index = 0
|
||||
pending = None
|
||||
while not self.stop.is_set():
|
||||
write = worker_id % 2 == 0
|
||||
if write and pending is None:
|
||||
key = uuid.uuid4().hex
|
||||
pending = (key, f"grade4-{worker_id}-{index}-{key}")
|
||||
index += 1
|
||||
with self.lock:
|
||||
self.attempted.add(pending[1])
|
||||
try:
|
||||
if write:
|
||||
response = client.message(self.actor, self.chat,
|
||||
pending[1], key=pending[0])
|
||||
else:
|
||||
response = client.call("GET", "/api/v1/users/me",
|
||||
actor=self.actor, schema="User")
|
||||
event = (time.monotonic(), "write" if write else "read", True,
|
||||
response["headers"]["X-Instance-Id"])
|
||||
if write:
|
||||
with self.lock:
|
||||
self.confirmed.add(pending[1])
|
||||
pending = None
|
||||
except Failure:
|
||||
event = (time.monotonic(), "write" if write else "read", False, None)
|
||||
with self.lock:
|
||||
self.events.append(event)
|
||||
self.stop.wait(0.05)
|
||||
|
||||
def finish(self):
|
||||
self.stop.set()
|
||||
try:
|
||||
for future in self.futures:
|
||||
future.result(timeout=5)
|
||||
finally:
|
||||
self.pool.shutdown(wait=True)
|
||||
|
||||
def snapshot(self):
|
||||
with self.lock:
|
||||
return list(self.events)
|
||||
|
||||
|
||||
def metrics(events, started, ended, target, action_done, scenario):
|
||||
window = [event for event in events if started <= event[0] <= ended]
|
||||
check(bool(window), "no traffic was recorded during the action")
|
||||
counts = {}
|
||||
for kind in ("read", "write"):
|
||||
records = [event for event in window if event[1] == kind]
|
||||
successes = [event for event in records if event[2]]
|
||||
check(bool(successes), f"no successful {kind}s during the action")
|
||||
survivor = next((event[0] for event in successes
|
||||
if event[0] >= started and event[3] != target), None)
|
||||
check(survivor is not None, f"no {kind} served by surviving replica")
|
||||
after_action = next((event[0] for event in successes
|
||||
if event[0] >= action_done), None)
|
||||
check(after_action is not None, f"no {kind} succeeded after the action")
|
||||
counts[kind] = {
|
||||
"successful": len(successes),
|
||||
"errors": len(records) - len(successes),
|
||||
"successful_rps": round(len(successes) / (ended - started), 3),
|
||||
"first_survivor_seconds": round(survivor - started, 3),
|
||||
"first_success_after_action_seconds": round(after_action - started, 3),
|
||||
}
|
||||
after = [event for event in window if event[0] >= action_done + 1 and event[2]]
|
||||
check(bool(after), "no successful traffic after the action")
|
||||
if scenario == "failover":
|
||||
check(all(event[3] != target for event in after),
|
||||
"stopped replica still served traffic after the action")
|
||||
target_times = [event[0] for event in window
|
||||
if event[2] and event[3] == target]
|
||||
counts["last_target_response_seconds"] = (
|
||||
round(max(target_times) - started, 3) if target_times else None
|
||||
)
|
||||
for kind in ("read", "write"):
|
||||
check(counts[kind]["first_success_after_action_seconds"] <= 10,
|
||||
f"{kind} did not recover within 10 seconds")
|
||||
return counts
|
||||
|
||||
|
||||
def verify_history(client, actor, chat, original, traffic):
|
||||
messages = history(client, actor, chat)
|
||||
counts = Counter(message["text"] for message in messages)
|
||||
check(any(message == original for message in messages),
|
||||
"confirmed pre-action message disappeared")
|
||||
for text in traffic.attempted:
|
||||
check(counts[text] <= 1, "an uncertain retry created a duplicate")
|
||||
for text in traffic.confirmed:
|
||||
check(counts[text] == 1, "confirmed write disappeared")
|
||||
|
||||
|
||||
def resource_limits():
|
||||
expected = {
|
||||
"nginx": (500_000_000, 128 * 1024**2),
|
||||
"migrate": (1_000_000_000, 512 * 1024**2),
|
||||
"postgres": (1_000_000_000, 512 * 1024**2),
|
||||
"redis": (500_000_000, 256 * 1024**2),
|
||||
}
|
||||
ids = containers("api")
|
||||
check(len(ids) in (1, 2, 4), "expected 1, 2 or 4 API replicas")
|
||||
api_limits = (2_000_000_000 // len(ids), 1024 * 1024**2 // len(ids))
|
||||
observed = {"api": []}
|
||||
for container_id in ids:
|
||||
actual = (
|
||||
int(inspect(container_id, "{{.HostConfig.NanoCpus}}")),
|
||||
int(inspect(container_id, "{{.HostConfig.Memory}}")),
|
||||
)
|
||||
check(actual == api_limits, "API resource limits differ from fixed budget")
|
||||
observed["api"].append({"instance": instance(container_id),
|
||||
"cpus": actual[0] / 1e9,
|
||||
"memory_mib": actual[1] // 1024**2})
|
||||
for service, limits in expected.items():
|
||||
container_id = container(service)
|
||||
actual = (
|
||||
int(inspect(container_id, "{{.HostConfig.NanoCpus}}")),
|
||||
int(inspect(container_id, "{{.HostConfig.Memory}}")),
|
||||
)
|
||||
check(actual == limits, f"{service} resource limits differ from compose.yaml")
|
||||
observed[service] = {"cpus": actual[0] / 1e9,
|
||||
"memory_mib": actual[1] // 1024**2}
|
||||
return observed
|
||||
|
||||
|
||||
def run(args):
|
||||
check(2 <= args.concurrency <= 8, "concurrency must be 2..8")
|
||||
check(3 <= args.observe_seconds <= 60,
|
||||
"observe-seconds must be 3..60")
|
||||
result = {"scenario": args.scenario, "resources": resource_limits(),
|
||||
"actions": []}
|
||||
if args.scenario == "resources":
|
||||
return result
|
||||
targets = containers("api")
|
||||
check(len(targets) == 2, "failover and rolling scenarios require the two-replica profile")
|
||||
|
||||
client = Client(args.base_url, timeout=10)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
original = client.message(alice, chat, "Before grade 4 actions")["json"]
|
||||
both_observed(client, alice, {instance(item) for item in targets})
|
||||
migrate_id = container("migrate")
|
||||
traffic = Traffic(args.base_url, alice, chat, args.concurrency)
|
||||
traffic.start()
|
||||
try:
|
||||
for before_id in targets:
|
||||
target = instance(before_id)
|
||||
healthy(before_id)
|
||||
before_ids = set(containers("api"))
|
||||
both_observed(client, alice, {instance(item) for item in before_ids})
|
||||
time.sleep(1)
|
||||
started = time.monotonic()
|
||||
if args.scenario == "failover":
|
||||
docker("stop", "--time", "30", before_id, timeout=40)
|
||||
check(inspect(before_id, "{{.State.Status}}") == "exited",
|
||||
"target API container was not stopped")
|
||||
else:
|
||||
docker("stop", "--time", "30", before_id, timeout=40)
|
||||
docker("rm", before_id)
|
||||
compose("up", "-d", "--no-build", "--no-deps",
|
||||
"--scale", "api=2", "api", timeout=90)
|
||||
replacements = set(containers("api")) - before_ids
|
||||
check(len(replacements) == 1, "one API container was not replaced")
|
||||
healthy(next(iter(replacements)))
|
||||
action_done = time.monotonic()
|
||||
time.sleep(args.observe_seconds)
|
||||
ended = time.monotonic()
|
||||
item = {"target": target,
|
||||
"action_seconds": round(action_done - started, 3),
|
||||
"window_seconds": round(ended - started, 3),
|
||||
"traffic": metrics(traffic.snapshot(), started, ended,
|
||||
target, action_done, args.scenario)}
|
||||
if args.scenario == "failover":
|
||||
docker("start", before_id)
|
||||
healthy(before_id)
|
||||
both_observed(client, alice,
|
||||
{instance(item) for item in containers("api")})
|
||||
result["actions"].append(item)
|
||||
check(container("migrate") == migrate_id,
|
||||
"migration container changed during API replacement")
|
||||
finally:
|
||||
traffic.finish()
|
||||
# Restore both API replicas after a failed test; never remove volumes.
|
||||
try:
|
||||
compose("up", "-d", "--no-build", "--no-deps",
|
||||
"--scale", "api=2", "api")
|
||||
except Failure:
|
||||
print("WARNING: restore API replicas manually", file=sys.stderr)
|
||||
verify_history(client, bob, chat, original, traffic)
|
||||
result["confirmed_writes"] = len(traffic.confirmed)
|
||||
result["attempted_unique_writes"] = len(traffic.attempted)
|
||||
result["migration_container_unchanged"] = True
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--scenario", choices=("resources", "failover", "rolling"),
|
||||
required=True)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--concurrency", type=int, default=4)
|
||||
parser.add_argument("--observe-seconds", type=float, default=10)
|
||||
parser.add_argument("--output", help="optional JSON result path without tokens")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
result = run(args)
|
||||
output = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
path = Path(args.output)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(output + "\n", encoding="utf-8")
|
||||
print(output)
|
||||
print("PASS: grade 4 scenario")
|
||||
except (Failure, ValueError, OSError) as exc:
|
||||
print(f"FAIL: {exc}", file=sys.stderr)
|
||||
raise SystemExit(1) from None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user