#!/usr/bin/env python3 """Lab 3 grade 4: measured replica failover or sequential replacement.""" import argparse from collections import Counter from concurrent.futures import ThreadPoolExecutor import json from pathlib import Path import subprocess import sys import threading import time import uuid ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure, check # noqa: E402 from resilience import history # noqa: E402 def compose(*args, timeout=60): try: result = subprocess.run( ["docker", "compose", "-f", "compose.yaml", "-f", "configs/two.yaml", *args], cwd=ROOT, capture_output=True, text=True, timeout=timeout, check=False, ) except (OSError, subprocess.TimeoutExpired) as exc: raise Failure(f"docker compose {args[0]} failed: {type(exc).__name__}") from None check(result.returncode == 0, f"docker compose {args[0]} failed; inspect Docker logs") return result.stdout.strip() def docker(*args, timeout=60): try: result = subprocess.run( ["docker", *args], cwd=ROOT, capture_output=True, text=True, timeout=timeout, check=False, ) except (OSError, subprocess.TimeoutExpired) as exc: raise Failure(f"docker {args[0]} failed: {type(exc).__name__}") from None check(result.returncode == 0, f"docker {args[0]} failed; inspect Docker logs") return result.stdout.strip() def inspect(container_id, field): try: result = subprocess.run( ["docker", "inspect", "--format", field, container_id], cwd=ROOT, capture_output=True, text=True, timeout=10, check=False, ) except (OSError, subprocess.TimeoutExpired) as exc: raise Failure(f"docker inspect failed: {type(exc).__name__}") from None check(result.returncode == 0, "docker inspect failed") return result.stdout.strip() def containers(service): ids = compose("ps", "--all", "-q", service).splitlines() check(bool(ids), f"{service} container is missing") return ids def container(service): ids = containers(service) check(len(ids) == 1, f"expected one {service} container") return ids[0] def instance(container_id): return inspect(container_id, "{{.Config.Hostname}}") def healthy(container_id, timeout=60): deadline = time.monotonic() + timeout while time.monotonic() < deadline: if inspect(container_id, "{{.State.Health.Status}}") == "healthy": return time.sleep(0.5) raise Failure(f"API container did not become healthy in {timeout} seconds") def both_observed(client, actor, expected, timeout=30): seen = set() deadline = time.monotonic() + timeout while time.monotonic() < deadline: try: response = client.call("GET", "/api/v1/users/me", actor=actor, schema="User") seen.add(response["headers"]["X-Instance-Id"]) except Failure: pass if seen == expected: return time.sleep(0.1) raise Failure(f"both replicas were not observed; saw {sorted(seen)}") class Traffic: def __init__(self, base_url, actor, chat, concurrency): self.base_url = base_url self.actor = actor self.chat = chat self.concurrency = concurrency self.stop = threading.Event() self.lock = threading.Lock() self.events = [] self.confirmed = set() self.attempted = set() self.pool = ThreadPoolExecutor(max_workers=concurrency) def start(self): self.futures = [self.pool.submit(self.worker, i) for i in range(self.concurrency)] def worker(self, worker_id): client = Client(self.base_url, timeout=2) index = 0 pending = None while not self.stop.is_set(): write = worker_id % 2 == 0 if write and pending is None: key = uuid.uuid4().hex pending = (key, f"grade4-{worker_id}-{index}-{key}") index += 1 with self.lock: self.attempted.add(pending[1]) try: if write: response = client.message(self.actor, self.chat, pending[1], key=pending[0]) else: response = client.call("GET", "/api/v1/users/me", actor=self.actor, schema="User") event = (time.monotonic(), "write" if write else "read", True, response["headers"]["X-Instance-Id"]) if write: with self.lock: self.confirmed.add(pending[1]) pending = None except Failure: event = (time.monotonic(), "write" if write else "read", False, None) with self.lock: self.events.append(event) self.stop.wait(0.05) def finish(self): self.stop.set() try: for future in self.futures: future.result(timeout=5) finally: self.pool.shutdown(wait=True) def snapshot(self): with self.lock: return list(self.events) def metrics(events, started, ended, target, action_done, scenario): window = [event for event in events if started <= event[0] <= ended] check(bool(window), "no traffic was recorded during the action") counts = {} for kind in ("read", "write"): records = [event for event in window if event[1] == kind] successes = [event for event in records if event[2]] check(bool(successes), f"no successful {kind}s during the action") survivor = next((event[0] for event in successes if event[0] >= started and event[3] != target), None) check(survivor is not None, f"no {kind} served by surviving replica") after_action = next((event[0] for event in successes if event[0] >= action_done), None) check(after_action is not None, f"no {kind} succeeded after the action") counts[kind] = { "successful": len(successes), "errors": len(records) - len(successes), "successful_rps": round(len(successes) / (ended - started), 3), "first_survivor_seconds": round(survivor - started, 3), "first_success_after_action_seconds": round(after_action - started, 3), } after = [event for event in window if event[0] >= action_done + 1 and event[2]] check(bool(after), "no successful traffic after the action") if scenario == "failover": check(all(event[3] != target for event in after), "stopped replica still served traffic after the action") target_times = [event[0] for event in window if event[2] and event[3] == target] counts["last_target_response_seconds"] = ( round(max(target_times) - started, 3) if target_times else None ) for kind in ("read", "write"): check(counts[kind]["first_success_after_action_seconds"] <= 10, f"{kind} did not recover within 10 seconds") return counts def verify_history(client, actor, chat, original, traffic): messages = history(client, actor, chat) counts = Counter(message["text"] for message in messages) check(any(message == original for message in messages), "confirmed pre-action message disappeared") for text in traffic.attempted: check(counts[text] <= 1, "an uncertain retry created a duplicate") for text in traffic.confirmed: check(counts[text] == 1, "confirmed write disappeared") def resource_limits(): expected = { "nginx": (500_000_000, 128 * 1024**2), "migrate": (1_000_000_000, 512 * 1024**2), "postgres": (1_000_000_000, 512 * 1024**2), "redis": (500_000_000, 256 * 1024**2), } ids = containers("api") check(len(ids) in (1, 2, 4), "expected 1, 2 or 4 API replicas") api_limits = (2_000_000_000 // len(ids), 1024 * 1024**2 // len(ids)) observed = {"api": []} for container_id in ids: actual = ( int(inspect(container_id, "{{.HostConfig.NanoCpus}}")), int(inspect(container_id, "{{.HostConfig.Memory}}")), ) check(actual == api_limits, "API resource limits differ from fixed budget") observed["api"].append({"instance": instance(container_id), "cpus": actual[0] / 1e9, "memory_mib": actual[1] // 1024**2}) for service, limits in expected.items(): container_id = container(service) actual = ( int(inspect(container_id, "{{.HostConfig.NanoCpus}}")), int(inspect(container_id, "{{.HostConfig.Memory}}")), ) check(actual == limits, f"{service} resource limits differ from compose.yaml") observed[service] = {"cpus": actual[0] / 1e9, "memory_mib": actual[1] // 1024**2} return observed def run(args): check(2 <= args.concurrency <= 8, "concurrency must be 2..8") check(3 <= args.observe_seconds <= 60, "observe-seconds must be 3..60") result = {"scenario": args.scenario, "resources": resource_limits(), "actions": []} if args.scenario == "resources": return result targets = containers("api") check(len(targets) == 2, "failover and rolling scenarios require the two-replica profile") client = Client(args.base_url, timeout=10) alice, bob, _, chat = client.bootstrap() original = client.message(alice, chat, "Before grade 4 actions")["json"] both_observed(client, alice, {instance(item) for item in targets}) migrate_id = container("migrate") traffic = Traffic(args.base_url, alice, chat, args.concurrency) traffic.start() try: for before_id in targets: target = instance(before_id) healthy(before_id) before_ids = set(containers("api")) both_observed(client, alice, {instance(item) for item in before_ids}) time.sleep(1) started = time.monotonic() if args.scenario == "failover": docker("stop", "--time", "30", before_id, timeout=40) check(inspect(before_id, "{{.State.Status}}") == "exited", "target API container was not stopped") else: docker("stop", "--time", "30", before_id, timeout=40) docker("rm", before_id) compose("up", "-d", "--no-build", "--no-deps", "--scale", "api=2", "api", timeout=90) replacements = set(containers("api")) - before_ids check(len(replacements) == 1, "one API container was not replaced") healthy(next(iter(replacements))) action_done = time.monotonic() time.sleep(args.observe_seconds) ended = time.monotonic() item = {"target": target, "action_seconds": round(action_done - started, 3), "window_seconds": round(ended - started, 3), "traffic": metrics(traffic.snapshot(), started, ended, target, action_done, args.scenario)} if args.scenario == "failover": docker("start", before_id) healthy(before_id) both_observed(client, alice, {instance(item) for item in containers("api")}) result["actions"].append(item) check(container("migrate") == migrate_id, "migration container changed during API replacement") finally: traffic.finish() # Restore both API replicas after a failed test; never remove volumes. try: compose("up", "-d", "--no-build", "--no-deps", "--scale", "api=2", "api") except Failure: print("WARNING: restore API replicas manually", file=sys.stderr) verify_history(client, bob, chat, original, traffic) result["confirmed_writes"] = len(traffic.confirmed) result["attempted_unique_writes"] = len(traffic.attempted) result["migration_container_unchanged"] = True return result def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--scenario", choices=("resources", "failover", "rolling"), required=True) parser.add_argument("--base-url", default="http://127.0.0.1:8080") parser.add_argument("--concurrency", type=int, default=4) parser.add_argument("--observe-seconds", type=float, default=10) parser.add_argument("--output", help="optional JSON result path without tokens") args = parser.parse_args() try: result = run(args) output = json.dumps(result, ensure_ascii=False, indent=2) if args.output: path = Path(args.output) path.parent.mkdir(parents=True, exist_ok=True) path.write_text(output + "\n", encoding="utf-8") print(output) print("PASS: grade 4 scenario") except (Failure, ValueError, OSError) as exc: print(f"FAIL: {exc}", file=sys.stderr) raise SystemExit(1) from None if __name__ == "__main__": main()