Files
mtusi2026lab3/scripts/grade4_experiment.py
T
2026-10-25 21:46:21 +03:00

398 lines
14 KiB
Python
Executable File

#!/usr/bin/env python3
"""Run the repeatable, secret-free resilience experiment required for grade 4."""
import argparse
from collections import Counter, defaultdict
import datetime as dt
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
def command(args, env=None, timeout=240, check=True):
result = subprocess.run(
[str(value) for value in args],
cwd=ROOT,
env=env,
text=True,
capture_output=True,
timeout=timeout,
check=False,
)
if check and result.returncode:
raise RuntimeError(f"{' '.join(map(str, args[:3]))} exited {result.returncode}")
return result
def compose(env_file, *args):
return command(["docker", "compose", "--env-file", env_file, *args])
def api_ids(env_file):
return [line for line in compose(env_file, "ps", "-q", "api").stdout.splitlines() if line]
def inspect(container_id):
data = command(["docker", "inspect", "--format", "{{json .}}", container_id]).stdout
return json.loads(data)
def wait_api(env_file, expected, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
healthy = []
for container_id in api_ids(env_file):
state = inspect(container_id)["State"]
if state.get("Health", {}).get("Status") == "healthy":
healthy.append(container_id)
if len(healthy) >= expected:
return healthy
time.sleep(0.5)
raise RuntimeError(f"fewer than {expected} API replicas became healthy")
def percentile(values, ratio):
if not values:
return None
values = sorted(values)
return round(values[max(0, math.ceil(ratio * len(values)) - 1)], 3)
def memory_mib(raw):
value = raw.split("/", 1)[0].strip()
units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
for suffix in ("GiB", "MiB", "KiB", "B"):
if value.endswith(suffix):
return round(float(value[:-len(suffix)]) * units[suffix] / 1024**2, 3)
return 0.0
def service(name):
for candidate in ("load-balancer", "frontend", "postgres", "redis", "api"):
if f"-{candidate}-" in name:
return candidate
return "other"
def sample_stats():
result = command(
["docker", "stats", "--no-stream", "--format", "{{json .}}"],
timeout=10,
check=False,
)
samples = []
if result.returncode:
return samples
for line in result.stdout.splitlines():
try:
item = json.loads(line)
samples.append({
"service": service(item.get("Name", "")),
"cpu_percent": float(item.get("CPUPerc", "0%").rstrip("%")),
"memory_mib": memory_mib(item.get("MemUsage", "0B / 0B")),
})
except (ValueError, json.JSONDecodeError):
continue
return samples
def aggregate_resources(samples):
grouped = defaultdict(lambda: {"cpu": [], "memory": []})
for item in samples:
if item["service"] != "other":
grouped[item["service"]]["cpu"].append(item["cpu_percent"])
grouped[item["service"]]["memory"].append(item["memory_mib"])
return {
name: {
"cpu_percent_avg": round(sum(values["cpu"]) / len(values["cpu"]), 3),
"cpu_percent_max": round(max(values["cpu"]), 3),
"memory_mib_avg": round(sum(values["memory"]) / len(values["memory"]), 3),
"memory_mib_max": round(max(values["memory"]), 3),
}
for name, values in sorted(grouped.items())
}
def load_phase(name, actor, chat, args, action=None, target=None):
records, resources = [], []
lock = threading.Lock()
stop = threading.Event()
barrier = threading.Barrier(args.concurrency)
started = time.monotonic()
def worker(worker_id):
client = Client(args.base_url, timeout=4)
index = 0
barrier.wait()
while not stop.is_set():
request_started = time.monotonic()
try:
response = client.message(actor, chat, f"grade4-{name}-{worker_id}-{index}")
row = {
"t": round(time.monotonic() - started, 3),
"ok": True,
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
"instance": response["headers"].get("X-Instance-Id", ""),
}
except Failure as exc:
row = {
"t": round(time.monotonic() - started, 3),
"ok": False,
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
"error": str(exc),
}
with lock:
records.append(row)
index += 1
def sampler():
while not stop.is_set():
resources.extend(sample_stats())
stop.wait(1)
workers = [threading.Thread(target=worker, args=(index,), daemon=True)
for index in range(args.concurrency)]
stats_thread = threading.Thread(target=sampler, daemon=True)
stats_thread.start()
for thread in workers:
thread.start()
action_data = None
action_error = None
try:
if action is None:
time.sleep(args.baseline_seconds)
else:
time.sleep(args.action_after_seconds)
action_started = time.monotonic()
try:
summary = action()
exit_code = 0
except Exception as exc:
summary, exit_code, action_error = type(exc).__name__, 1, exc
action_ended = time.monotonic()
action_data = {
"started_at": round(action_started - started, 3),
"ended_at": round(action_ended - started, 3),
"duration_seconds": round(action_ended - action_started, 3),
"exit_code": exit_code,
"summary": summary,
}
time.sleep(args.post_action_seconds)
finally:
stop.set()
for thread in workers:
thread.join(10)
stats_thread.join(5)
if action_error:
raise action_error
elapsed = time.monotonic() - started
successes = [row for row in records if row["ok"]]
failures = [row for row in records if not row["ok"]]
latencies = [row["latency_ms"] for row in successes]
buckets = defaultdict(lambda: {"successful": 0, "failed": 0})
for row in records:
buckets[str(int(row["t"]))]["successful" if row["ok"] else "failed"] += 1
recovery = None
if action_data and target:
recovery_row = next((
row for row in successes
if row["t"] >= action_data["ended_at"] and row.get("instance") != target
), None)
if recovery_row:
recovery = round(recovery_row["t"] - action_data["started_at"], 3)
total = len(records)
return {
"name": name,
"elapsed_seconds": round(elapsed, 3),
"successful_requests": len(successes),
"failed_requests": len(failures),
"error_ratio": round(len(failures) / total, 6) if total else None,
"errors": dict(sorted(Counter(row["error"] for row in failures).items())),
"successful_rps": round(len(successes) / elapsed, 3),
"successful_latency_ms": {
"p50": percentile(latencies, 0.50),
"p95": percentile(latencies, 0.95),
"p99": percentile(latencies, 0.99),
},
"instances": sorted({row.get("instance") for row in successes if row.get("instance")}),
"recovery_seconds": recovery,
"action": action_data,
"per_second": dict(sorted(buckets.items(), key=lambda item: int(item[0]))),
"resources": aggregate_resources(resources),
}
def idempotency(actor, chat, base_url, count=32):
key, body = uuid.uuid4().hex, "grade4-idempotency-" + uuid.uuid4().hex
barrier, lock = threading.Barrier(count), threading.Lock()
rows, errors = [], []
def request():
client = Client(base_url, timeout=8)
barrier.wait()
try:
response = client.message(actor, chat, body, key=key)
row = (response["json"]["id"], response["headers"].get("X-Instance-Id", ""))
with lock:
rows.append(row)
except Failure as exc:
with lock:
errors.append(str(exc))
threads = [threading.Thread(target=request) for _ in range(count)]
for thread in threads:
thread.start()
for thread in threads:
thread.join(30)
message_ids = {row[0] for row in rows}
if errors or len(rows) != count or len(message_ids) != 1:
raise RuntimeError("parallel idempotency check failed")
return {
"parallel_requests": count,
"successful_responses": len(rows),
"failed_responses": len(errors),
"unique_message_ids": len(message_ids),
"observed_instances": sorted({row[1] for row in rows}),
}
def environment(env_file):
mem_total_kib = 0
with open("/proc/meminfo", encoding="utf-8") as meminfo:
for line in meminfo:
if line.startswith("MemTotal:"):
mem_total_kib = int(line.split()[1])
break
limits = []
for container_id in compose(env_file, "ps", "-q").stdout.splitlines():
data = inspect(container_id)
limits.append({
"service": service(data["Name"].lstrip("/")),
"nano_cpus": data["HostConfig"].get("NanoCpus", 0),
"memory_limit_bytes": data["HostConfig"].get("Memory", 0),
})
return {
"utc_started_at": dt.datetime.now(dt.timezone.utc).isoformat(),
"git_commit": command(["git", "rev-parse", "HEAD"]).stdout.strip(),
"working_tree_dirty": bool(command(["git", "status", "--porcelain"]).stdout.strip()),
"logical_cpu_count": os.cpu_count(),
"host_memory_mib": round(mem_total_kib / 1024, 1),
"docker_engine": command(["docker", "version", "--format", "{{.Server.Version}}"]).stdout.strip(),
"docker_compose": command(["docker", "compose", "version", "--short"]).stdout.strip(),
"container_limits": limits,
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--env-file", default=".env")
parser.add_argument("--concurrency", type=int, default=8)
parser.add_argument("--baseline-seconds", type=float, default=12)
parser.add_argument("--action-after-seconds", type=float, default=3)
parser.add_argument("--post-action-seconds", type=float, default=8)
parser.add_argument("--output", default="evidence/grade4/results.json")
args = parser.parse_args()
if not 1 <= args.concurrency <= 64:
raise SystemExit("concurrency must be 1..64")
original = wait_api(args.env_file, 2)
if len(original) != 2:
raise RuntimeError("experiment requires exactly two initial API replicas")
client = Client(args.base_url, timeout=5)
alice, _, _, chat = client.bootstrap()
for index in range(12):
client.message(alice, chat, f"grade4-warmup-{index}")
phases = [load_phase("baseline", alice, chat, args)]
targets = [(container_id, inspect(container_id)["Config"]["Hostname"])
for container_id in original]
for ordinal, (container_id, target) in enumerate(targets, start=1):
def stop_target(instance=target):
hook_env = dict(os.environ)
hook_env.update({"ENV_FILE": args.env_file, "TARGET_INSTANCE_ID": instance})
result = command([ROOT / "scripts" / "stop-one.sh"], env=hook_env, timeout=30)
return result.stdout.strip().splitlines()[-1]
phases.append(load_phase(
f"failover-{ordinal}", alice, chat, args,
action=stop_target, target=target,
))
command(["docker", "start", container_id])
wait_api(args.env_file, 2)
idem = idempotency(alice, chat, args.base_url)
def rolling():
hook_env = dict(os.environ)
hook_env.update({"ENV_FILE": args.env_file, "DESIRED_REPLICAS": "2"})
result = command(
[ROOT / "scripts" / "rolling-update.sh"],
env=hook_env,
timeout=240,
)
return result.stdout.strip().splitlines()[-1]
phases.append(load_phase("rolling-update", alice, chat, args, action=rolling))
wait_api(args.env_file, 2)
report = {
"schema_version": 1,
"environment": environment(args.env_file),
"methodology": {
"base_url": args.base_url,
"concurrency": args.concurrency,
"baseline_seconds": args.baseline_seconds,
"action_after_seconds": args.action_after_seconds,
"post_action_seconds": args.post_action_seconds,
"warmup_requests": 12,
"workload": "closed-loop POST messages via the public HAProxy gateway; no automatic retries",
},
"phases": phases,
"idempotency": idem,
"dataset": {
"warmup_requests": 12,
"minimum_confirmed_messages": 13 + sum(
phase["successful_requests"] for phase in phases
),
},
"limitations": [
"single Docker host",
"closed-loop coordinated omission",
"successful-response latency percentiles only",
"PostgreSQL and HAProxy remain single points of failure",
],
}
destination = ROOT / args.output
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
try:
displayed_output = destination.relative_to(ROOT)
except ValueError:
displayed_output = destination
print(json.dumps({
"output": str(displayed_output),
"phases": [{
"name": phase["name"],
"successful_rps": phase["successful_rps"],
"failed_requests": phase["failed_requests"],
"recovery_seconds": phase["recovery_seconds"],
} for phase in phases],
"idempotency": idem,
}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()