#!/usr/bin/env python3 """Run the repeatable, secret-free resilience experiment required for grade 4.""" import argparse from collections import Counter, defaultdict import datetime as dt import json import math import os from pathlib import Path import subprocess import sys import threading import time import uuid ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure # noqa: E402 def command(args, env=None, timeout=240, check=True): result = subprocess.run( [str(value) for value in args], cwd=ROOT, env=env, text=True, capture_output=True, timeout=timeout, check=False, ) if check and result.returncode: raise RuntimeError(f"{' '.join(map(str, args[:3]))} exited {result.returncode}") return result def compose(env_file, *args): return command(["docker", "compose", "--env-file", env_file, *args]) def api_ids(env_file): return [line for line in compose(env_file, "ps", "-q", "api").stdout.splitlines() if line] def inspect(container_id): data = command(["docker", "inspect", "--format", "{{json .}}", container_id]).stdout return json.loads(data) def wait_api(env_file, expected, timeout=90): deadline = time.monotonic() + timeout while time.monotonic() < deadline: healthy = [] for container_id in api_ids(env_file): state = inspect(container_id)["State"] if state.get("Health", {}).get("Status") == "healthy": healthy.append(container_id) if len(healthy) >= expected: return healthy time.sleep(0.5) raise RuntimeError(f"fewer than {expected} API replicas became healthy") def percentile(values, ratio): if not values: return None values = sorted(values) return round(values[max(0, math.ceil(ratio * len(values)) - 1)], 3) def memory_mib(raw): value = raw.split("/", 1)[0].strip() units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3} for suffix in ("GiB", "MiB", "KiB", "B"): if value.endswith(suffix): return round(float(value[:-len(suffix)]) * units[suffix] / 1024**2, 3) return 0.0 def service(name): for candidate in ("load-balancer", "postgres", "redis", "api"): if f"-{candidate}-" in name: return candidate return "other" def sample_stats(): result = command( ["docker", "stats", "--no-stream", "--format", "{{json .}}"], timeout=10, check=False, ) samples = [] if result.returncode: return samples for line in result.stdout.splitlines(): try: item = json.loads(line) samples.append({ "service": service(item.get("Name", "")), "cpu_percent": float(item.get("CPUPerc", "0%").rstrip("%")), "memory_mib": memory_mib(item.get("MemUsage", "0B / 0B")), }) except (ValueError, json.JSONDecodeError): continue return samples def aggregate_resources(samples): grouped = defaultdict(lambda: {"cpu": [], "memory": []}) for item in samples: if item["service"] != "other": grouped[item["service"]]["cpu"].append(item["cpu_percent"]) grouped[item["service"]]["memory"].append(item["memory_mib"]) return { name: { "cpu_percent_avg": round(sum(values["cpu"]) / len(values["cpu"]), 3), "cpu_percent_max": round(max(values["cpu"]), 3), "memory_mib_avg": round(sum(values["memory"]) / len(values["memory"]), 3), "memory_mib_max": round(max(values["memory"]), 3), } for name, values in sorted(grouped.items()) } def load_phase(name, actor, chat, args, action=None, target=None): records, resources = [], [] lock = threading.Lock() stop = threading.Event() barrier = threading.Barrier(args.concurrency) started = time.monotonic() def worker(worker_id): client = Client(args.base_url, timeout=4) index = 0 barrier.wait() while not stop.is_set(): request_started = time.monotonic() try: response = client.message(actor, chat, f"grade4-{name}-{worker_id}-{index}") row = { "t": round(time.monotonic() - started, 3), "ok": True, "latency_ms": round((time.monotonic() - request_started) * 1000, 3), "instance": response["headers"].get("X-Instance-Id", ""), } except Failure as exc: row = { "t": round(time.monotonic() - started, 3), "ok": False, "latency_ms": round((time.monotonic() - request_started) * 1000, 3), "error": str(exc), } with lock: records.append(row) index += 1 def sampler(): while not stop.is_set(): resources.extend(sample_stats()) stop.wait(1) workers = [threading.Thread(target=worker, args=(index,), daemon=True) for index in range(args.concurrency)] stats_thread = threading.Thread(target=sampler, daemon=True) stats_thread.start() for thread in workers: thread.start() action_data = None action_error = None try: if action is None: time.sleep(args.baseline_seconds) else: time.sleep(args.action_after_seconds) action_started = time.monotonic() try: summary = action() exit_code = 0 except Exception as exc: summary, exit_code, action_error = type(exc).__name__, 1, exc action_ended = time.monotonic() action_data = { "started_at": round(action_started - started, 3), "ended_at": round(action_ended - started, 3), "duration_seconds": round(action_ended - action_started, 3), "exit_code": exit_code, "summary": summary, } time.sleep(args.post_action_seconds) finally: stop.set() for thread in workers: thread.join(10) stats_thread.join(5) if action_error: raise action_error elapsed = time.monotonic() - started successes = [row for row in records if row["ok"]] failures = [row for row in records if not row["ok"]] latencies = [row["latency_ms"] for row in successes] buckets = defaultdict(lambda: {"successful": 0, "failed": 0}) for row in records: buckets[str(int(row["t"]))]["successful" if row["ok"] else "failed"] += 1 recovery = None if action_data and target: recovery_row = next(( row for row in successes if row["t"] >= action_data["ended_at"] and row.get("instance") != target ), None) if recovery_row: recovery = round(recovery_row["t"] - action_data["started_at"], 3) total = len(records) return { "name": name, "elapsed_seconds": round(elapsed, 3), "successful_requests": len(successes), "failed_requests": len(failures), "error_ratio": round(len(failures) / total, 6) if total else None, "errors": dict(sorted(Counter(row["error"] for row in failures).items())), "successful_rps": round(len(successes) / elapsed, 3), "successful_latency_ms": { "p50": percentile(latencies, 0.50), "p95": percentile(latencies, 0.95), "p99": percentile(latencies, 0.99), }, "instances": sorted({row.get("instance") for row in successes if row.get("instance")}), "recovery_seconds": recovery, "action": action_data, "per_second": dict(sorted(buckets.items(), key=lambda item: int(item[0]))), "resources": aggregate_resources(resources), } def idempotency(actor, chat, base_url, count=32): key, body = uuid.uuid4().hex, "grade4-idempotency-" + uuid.uuid4().hex barrier, lock = threading.Barrier(count), threading.Lock() rows, errors = [], [] def request(): client = Client(base_url, timeout=8) barrier.wait() try: response = client.message(actor, chat, body, key=key) row = (response["json"]["id"], response["headers"].get("X-Instance-Id", "")) with lock: rows.append(row) except Failure as exc: with lock: errors.append(str(exc)) threads = [threading.Thread(target=request) for _ in range(count)] for thread in threads: thread.start() for thread in threads: thread.join(30) message_ids = {row[0] for row in rows} if errors or len(rows) != count or len(message_ids) != 1: raise RuntimeError("parallel idempotency check failed") return { "parallel_requests": count, "successful_responses": len(rows), "failed_responses": len(errors), "unique_message_ids": len(message_ids), "observed_instances": sorted({row[1] for row in rows}), } def environment(env_file): mem_total_kib = 0 with open("/proc/meminfo", encoding="utf-8") as meminfo: for line in meminfo: if line.startswith("MemTotal:"): mem_total_kib = int(line.split()[1]) break limits = [] for container_id in compose(env_file, "ps", "-q").stdout.splitlines(): data = inspect(container_id) limits.append({ "service": service(data["Name"].lstrip("/")), "nano_cpus": data["HostConfig"].get("NanoCpus", 0), "memory_limit_bytes": data["HostConfig"].get("Memory", 0), }) return { "utc_started_at": dt.datetime.now(dt.timezone.utc).isoformat(), "git_commit": command(["git", "rev-parse", "HEAD"]).stdout.strip(), "working_tree_dirty": bool(command(["git", "status", "--porcelain"]).stdout.strip()), "logical_cpu_count": os.cpu_count(), "host_memory_mib": round(mem_total_kib / 1024, 1), "docker_engine": command(["docker", "version", "--format", "{{.Server.Version}}"]).stdout.strip(), "docker_compose": command(["docker", "compose", "version", "--short"]).stdout.strip(), "container_limits": limits, } def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-url", default="http://localhost:8080") parser.add_argument("--env-file", default=".env") parser.add_argument("--concurrency", type=int, default=8) parser.add_argument("--baseline-seconds", type=float, default=12) parser.add_argument("--action-after-seconds", type=float, default=3) parser.add_argument("--post-action-seconds", type=float, default=8) parser.add_argument("--output", default="evidence/grade4/results.json") args = parser.parse_args() if not 1 <= args.concurrency <= 64: raise SystemExit("concurrency must be 1..64") original = wait_api(args.env_file, 2) if len(original) != 2: raise RuntimeError("experiment requires exactly two initial API replicas") client = Client(args.base_url, timeout=5) alice, _, _, chat = client.bootstrap() for index in range(12): client.message(alice, chat, f"grade4-warmup-{index}") phases = [load_phase("baseline", alice, chat, args)] targets = [(container_id, inspect(container_id)["Config"]["Hostname"]) for container_id in original] for ordinal, (container_id, target) in enumerate(targets, start=1): def stop_target(instance=target): hook_env = dict(os.environ) hook_env.update({"ENV_FILE": args.env_file, "TARGET_INSTANCE_ID": instance}) result = command([ROOT / "scripts" / "stop-one.sh"], env=hook_env, timeout=30) return result.stdout.strip().splitlines()[-1] phases.append(load_phase( f"failover-{ordinal}", alice, chat, args, action=stop_target, target=target, )) command(["docker", "start", container_id]) wait_api(args.env_file, 2) idem = idempotency(alice, chat, args.base_url) def rolling(): hook_env = dict(os.environ) hook_env.update({"ENV_FILE": args.env_file, "DESIRED_REPLICAS": "2"}) result = command( [ROOT / "scripts" / "rolling-update.sh"], env=hook_env, timeout=240, ) return result.stdout.strip().splitlines()[-1] phases.append(load_phase("rolling-update", alice, chat, args, action=rolling)) wait_api(args.env_file, 2) report = { "schema_version": 1, "environment": environment(args.env_file), "methodology": { "base_url": args.base_url, "concurrency": args.concurrency, "baseline_seconds": args.baseline_seconds, "action_after_seconds": args.action_after_seconds, "post_action_seconds": args.post_action_seconds, "warmup_requests": 12, "workload": "closed-loop POST messages via Nginx; no automatic retries", }, "phases": phases, "idempotency": idem, "dataset": { "warmup_requests": 12, "minimum_confirmed_messages": 13 + sum( phase["successful_requests"] for phase in phases ), }, "limitations": [ "single Docker host", "closed-loop coordinated omission", "successful-response latency percentiles only", "PostgreSQL and Nginx remain single points of failure", ], } destination = ROOT / args.output destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") try: displayed_output = destination.relative_to(ROOT) except ValueError: displayed_output = destination print(json.dumps({ "output": str(displayed_output), "phases": [{ "name": phase["name"], "successful_rps": phase["successful_rps"], "failed_requests": phase["failed_requests"], "recovery_seconds": phase["recovery_seconds"], } for phase in phases], "idempotency": idem, }, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()