forked from mtusi2026/mtusi2026lab3
398 lines
14 KiB
Python
Executable File
398 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Run the repeatable, secret-free resilience experiment required for grade 4."""
|
|
import argparse
|
|
from collections import Counter, defaultdict
|
|
import datetime as dt
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure # noqa: E402
|
|
|
|
|
|
def command(args, env=None, timeout=240, check=True):
|
|
result = subprocess.run(
|
|
[str(value) for value in args],
|
|
cwd=ROOT,
|
|
env=env,
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=timeout,
|
|
check=False,
|
|
)
|
|
if check and result.returncode:
|
|
raise RuntimeError(f"{' '.join(map(str, args[:3]))} exited {result.returncode}")
|
|
return result
|
|
|
|
|
|
def compose(env_file, *args):
|
|
return command(["docker", "compose", "--env-file", env_file, *args])
|
|
|
|
|
|
def api_ids(env_file):
|
|
return [line for line in compose(env_file, "ps", "-q", "api").stdout.splitlines() if line]
|
|
|
|
|
|
def inspect(container_id):
|
|
data = command(["docker", "inspect", "--format", "{{json .}}", container_id]).stdout
|
|
return json.loads(data)
|
|
|
|
|
|
def wait_api(env_file, expected, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
healthy = []
|
|
for container_id in api_ids(env_file):
|
|
state = inspect(container_id)["State"]
|
|
if state.get("Health", {}).get("Status") == "healthy":
|
|
healthy.append(container_id)
|
|
if len(healthy) >= expected:
|
|
return healthy
|
|
time.sleep(0.5)
|
|
raise RuntimeError(f"fewer than {expected} API replicas became healthy")
|
|
|
|
|
|
def percentile(values, ratio):
|
|
if not values:
|
|
return None
|
|
values = sorted(values)
|
|
return round(values[max(0, math.ceil(ratio * len(values)) - 1)], 3)
|
|
|
|
|
|
def memory_mib(raw):
|
|
value = raw.split("/", 1)[0].strip()
|
|
units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
|
|
for suffix in ("GiB", "MiB", "KiB", "B"):
|
|
if value.endswith(suffix):
|
|
return round(float(value[:-len(suffix)]) * units[suffix] / 1024**2, 3)
|
|
return 0.0
|
|
|
|
|
|
def service(name):
|
|
for candidate in ("load-balancer", "postgres", "redis", "api"):
|
|
if f"-{candidate}-" in name:
|
|
return candidate
|
|
return "other"
|
|
|
|
|
|
def sample_stats():
|
|
result = command(
|
|
["docker", "stats", "--no-stream", "--format", "{{json .}}"],
|
|
timeout=10,
|
|
check=False,
|
|
)
|
|
samples = []
|
|
if result.returncode:
|
|
return samples
|
|
for line in result.stdout.splitlines():
|
|
try:
|
|
item = json.loads(line)
|
|
samples.append({
|
|
"service": service(item.get("Name", "")),
|
|
"cpu_percent": float(item.get("CPUPerc", "0%").rstrip("%")),
|
|
"memory_mib": memory_mib(item.get("MemUsage", "0B / 0B")),
|
|
})
|
|
except (ValueError, json.JSONDecodeError):
|
|
continue
|
|
return samples
|
|
|
|
|
|
def aggregate_resources(samples):
|
|
grouped = defaultdict(lambda: {"cpu": [], "memory": []})
|
|
for item in samples:
|
|
if item["service"] != "other":
|
|
grouped[item["service"]]["cpu"].append(item["cpu_percent"])
|
|
grouped[item["service"]]["memory"].append(item["memory_mib"])
|
|
return {
|
|
name: {
|
|
"cpu_percent_avg": round(sum(values["cpu"]) / len(values["cpu"]), 3),
|
|
"cpu_percent_max": round(max(values["cpu"]), 3),
|
|
"memory_mib_avg": round(sum(values["memory"]) / len(values["memory"]), 3),
|
|
"memory_mib_max": round(max(values["memory"]), 3),
|
|
}
|
|
for name, values in sorted(grouped.items())
|
|
}
|
|
|
|
|
|
def load_phase(name, actor, chat, args, action=None, target=None):
|
|
records, resources = [], []
|
|
lock = threading.Lock()
|
|
stop = threading.Event()
|
|
barrier = threading.Barrier(args.concurrency)
|
|
started = time.monotonic()
|
|
|
|
def worker(worker_id):
|
|
client = Client(args.base_url, timeout=4)
|
|
index = 0
|
|
barrier.wait()
|
|
while not stop.is_set():
|
|
request_started = time.monotonic()
|
|
try:
|
|
response = client.message(actor, chat, f"grade4-{name}-{worker_id}-{index}")
|
|
row = {
|
|
"t": round(time.monotonic() - started, 3),
|
|
"ok": True,
|
|
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
|
|
"instance": response["headers"].get("X-Instance-Id", ""),
|
|
}
|
|
except Failure as exc:
|
|
row = {
|
|
"t": round(time.monotonic() - started, 3),
|
|
"ok": False,
|
|
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
|
|
"error": str(exc),
|
|
}
|
|
with lock:
|
|
records.append(row)
|
|
index += 1
|
|
|
|
def sampler():
|
|
while not stop.is_set():
|
|
resources.extend(sample_stats())
|
|
stop.wait(1)
|
|
|
|
workers = [threading.Thread(target=worker, args=(index,), daemon=True)
|
|
for index in range(args.concurrency)]
|
|
stats_thread = threading.Thread(target=sampler, daemon=True)
|
|
stats_thread.start()
|
|
for thread in workers:
|
|
thread.start()
|
|
|
|
action_data = None
|
|
action_error = None
|
|
try:
|
|
if action is None:
|
|
time.sleep(args.baseline_seconds)
|
|
else:
|
|
time.sleep(args.action_after_seconds)
|
|
action_started = time.monotonic()
|
|
try:
|
|
summary = action()
|
|
exit_code = 0
|
|
except Exception as exc:
|
|
summary, exit_code, action_error = type(exc).__name__, 1, exc
|
|
action_ended = time.monotonic()
|
|
action_data = {
|
|
"started_at": round(action_started - started, 3),
|
|
"ended_at": round(action_ended - started, 3),
|
|
"duration_seconds": round(action_ended - action_started, 3),
|
|
"exit_code": exit_code,
|
|
"summary": summary,
|
|
}
|
|
time.sleep(args.post_action_seconds)
|
|
finally:
|
|
stop.set()
|
|
for thread in workers:
|
|
thread.join(10)
|
|
stats_thread.join(5)
|
|
if action_error:
|
|
raise action_error
|
|
|
|
elapsed = time.monotonic() - started
|
|
successes = [row for row in records if row["ok"]]
|
|
failures = [row for row in records if not row["ok"]]
|
|
latencies = [row["latency_ms"] for row in successes]
|
|
buckets = defaultdict(lambda: {"successful": 0, "failed": 0})
|
|
for row in records:
|
|
buckets[str(int(row["t"]))]["successful" if row["ok"] else "failed"] += 1
|
|
recovery = None
|
|
if action_data and target:
|
|
recovery_row = next((
|
|
row for row in successes
|
|
if row["t"] >= action_data["ended_at"] and row.get("instance") != target
|
|
), None)
|
|
if recovery_row:
|
|
recovery = round(recovery_row["t"] - action_data["started_at"], 3)
|
|
total = len(records)
|
|
return {
|
|
"name": name,
|
|
"elapsed_seconds": round(elapsed, 3),
|
|
"successful_requests": len(successes),
|
|
"failed_requests": len(failures),
|
|
"error_ratio": round(len(failures) / total, 6) if total else None,
|
|
"errors": dict(sorted(Counter(row["error"] for row in failures).items())),
|
|
"successful_rps": round(len(successes) / elapsed, 3),
|
|
"successful_latency_ms": {
|
|
"p50": percentile(latencies, 0.50),
|
|
"p95": percentile(latencies, 0.95),
|
|
"p99": percentile(latencies, 0.99),
|
|
},
|
|
"instances": sorted({row.get("instance") for row in successes if row.get("instance")}),
|
|
"recovery_seconds": recovery,
|
|
"action": action_data,
|
|
"per_second": dict(sorted(buckets.items(), key=lambda item: int(item[0]))),
|
|
"resources": aggregate_resources(resources),
|
|
}
|
|
|
|
|
|
def idempotency(actor, chat, base_url, count=32):
|
|
key, body = uuid.uuid4().hex, "grade4-idempotency-" + uuid.uuid4().hex
|
|
barrier, lock = threading.Barrier(count), threading.Lock()
|
|
rows, errors = [], []
|
|
|
|
def request():
|
|
client = Client(base_url, timeout=8)
|
|
barrier.wait()
|
|
try:
|
|
response = client.message(actor, chat, body, key=key)
|
|
row = (response["json"]["id"], response["headers"].get("X-Instance-Id", ""))
|
|
with lock:
|
|
rows.append(row)
|
|
except Failure as exc:
|
|
with lock:
|
|
errors.append(str(exc))
|
|
|
|
threads = [threading.Thread(target=request) for _ in range(count)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join(30)
|
|
message_ids = {row[0] for row in rows}
|
|
if errors or len(rows) != count or len(message_ids) != 1:
|
|
raise RuntimeError("parallel idempotency check failed")
|
|
return {
|
|
"parallel_requests": count,
|
|
"successful_responses": len(rows),
|
|
"failed_responses": len(errors),
|
|
"unique_message_ids": len(message_ids),
|
|
"observed_instances": sorted({row[1] for row in rows}),
|
|
}
|
|
|
|
|
|
def environment(env_file):
|
|
mem_total_kib = 0
|
|
with open("/proc/meminfo", encoding="utf-8") as meminfo:
|
|
for line in meminfo:
|
|
if line.startswith("MemTotal:"):
|
|
mem_total_kib = int(line.split()[1])
|
|
break
|
|
limits = []
|
|
for container_id in compose(env_file, "ps", "-q").stdout.splitlines():
|
|
data = inspect(container_id)
|
|
limits.append({
|
|
"service": service(data["Name"].lstrip("/")),
|
|
"nano_cpus": data["HostConfig"].get("NanoCpus", 0),
|
|
"memory_limit_bytes": data["HostConfig"].get("Memory", 0),
|
|
})
|
|
return {
|
|
"utc_started_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
"git_commit": command(["git", "rev-parse", "HEAD"]).stdout.strip(),
|
|
"working_tree_dirty": bool(command(["git", "status", "--porcelain"]).stdout.strip()),
|
|
"logical_cpu_count": os.cpu_count(),
|
|
"host_memory_mib": round(mem_total_kib / 1024, 1),
|
|
"docker_engine": command(["docker", "version", "--format", "{{.Server.Version}}"]).stdout.strip(),
|
|
"docker_compose": command(["docker", "compose", "version", "--short"]).stdout.strip(),
|
|
"container_limits": limits,
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--base-url", default="http://localhost:8080")
|
|
parser.add_argument("--env-file", default=".env")
|
|
parser.add_argument("--concurrency", type=int, default=8)
|
|
parser.add_argument("--baseline-seconds", type=float, default=12)
|
|
parser.add_argument("--action-after-seconds", type=float, default=3)
|
|
parser.add_argument("--post-action-seconds", type=float, default=8)
|
|
parser.add_argument("--output", default="evidence/grade4/results.json")
|
|
args = parser.parse_args()
|
|
if not 1 <= args.concurrency <= 64:
|
|
raise SystemExit("concurrency must be 1..64")
|
|
|
|
original = wait_api(args.env_file, 2)
|
|
if len(original) != 2:
|
|
raise RuntimeError("experiment requires exactly two initial API replicas")
|
|
client = Client(args.base_url, timeout=5)
|
|
alice, _, _, chat = client.bootstrap()
|
|
for index in range(12):
|
|
client.message(alice, chat, f"grade4-warmup-{index}")
|
|
|
|
phases = [load_phase("baseline", alice, chat, args)]
|
|
targets = [(container_id, inspect(container_id)["Config"]["Hostname"])
|
|
for container_id in original]
|
|
for ordinal, (container_id, target) in enumerate(targets, start=1):
|
|
def stop_target(instance=target):
|
|
hook_env = dict(os.environ)
|
|
hook_env.update({"ENV_FILE": args.env_file, "TARGET_INSTANCE_ID": instance})
|
|
result = command([ROOT / "scripts" / "stop-one.sh"], env=hook_env, timeout=30)
|
|
return result.stdout.strip().splitlines()[-1]
|
|
|
|
phases.append(load_phase(
|
|
f"failover-{ordinal}", alice, chat, args,
|
|
action=stop_target, target=target,
|
|
))
|
|
command(["docker", "start", container_id])
|
|
wait_api(args.env_file, 2)
|
|
|
|
idem = idempotency(alice, chat, args.base_url)
|
|
|
|
def rolling():
|
|
hook_env = dict(os.environ)
|
|
hook_env.update({"ENV_FILE": args.env_file, "DESIRED_REPLICAS": "2"})
|
|
result = command(
|
|
[ROOT / "scripts" / "rolling-update.sh"],
|
|
env=hook_env,
|
|
timeout=240,
|
|
)
|
|
return result.stdout.strip().splitlines()[-1]
|
|
|
|
phases.append(load_phase("rolling-update", alice, chat, args, action=rolling))
|
|
wait_api(args.env_file, 2)
|
|
|
|
report = {
|
|
"schema_version": 1,
|
|
"environment": environment(args.env_file),
|
|
"methodology": {
|
|
"base_url": args.base_url,
|
|
"concurrency": args.concurrency,
|
|
"baseline_seconds": args.baseline_seconds,
|
|
"action_after_seconds": args.action_after_seconds,
|
|
"post_action_seconds": args.post_action_seconds,
|
|
"warmup_requests": 12,
|
|
"workload": "closed-loop POST messages via Nginx; no automatic retries",
|
|
},
|
|
"phases": phases,
|
|
"idempotency": idem,
|
|
"dataset": {
|
|
"warmup_requests": 12,
|
|
"minimum_confirmed_messages": 13 + sum(
|
|
phase["successful_requests"] for phase in phases
|
|
),
|
|
},
|
|
"limitations": [
|
|
"single Docker host",
|
|
"closed-loop coordinated omission",
|
|
"successful-response latency percentiles only",
|
|
"PostgreSQL and Nginx remain single points of failure",
|
|
],
|
|
}
|
|
destination = ROOT / args.output
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
|
try:
|
|
displayed_output = destination.relative_to(ROOT)
|
|
except ValueError:
|
|
displayed_output = destination
|
|
print(json.dumps({
|
|
"output": str(displayed_output),
|
|
"phases": [{
|
|
"name": phase["name"],
|
|
"successful_rps": phase["successful_rps"],
|
|
"failed_requests": phase["failed_requests"],
|
|
"recovery_seconds": phase["recovery_seconds"],
|
|
} for phase in phases],
|
|
"idempotency": idem,
|
|
}, ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|