forked from mtusi2026/mtusi2026lab3
на 5
This commit is contained in:
@@ -77,7 +77,7 @@ def memory_mib(raw):
|
||||
|
||||
|
||||
def service(name):
|
||||
for candidate in ("load-balancer", "postgres", "redis", "api"):
|
||||
for candidate in ("load-balancer", "frontend", "postgres", "redis", "api"):
|
||||
if f"-{candidate}-" in name:
|
||||
return candidate
|
||||
return "other"
|
||||
@@ -357,7 +357,7 @@ def main():
|
||||
"action_after_seconds": args.action_after_seconds,
|
||||
"post_action_seconds": args.post_action_seconds,
|
||||
"warmup_requests": 12,
|
||||
"workload": "closed-loop POST messages via Nginx; no automatic retries",
|
||||
"workload": "closed-loop POST messages via the public HAProxy gateway; no automatic retries",
|
||||
},
|
||||
"phases": phases,
|
||||
"idempotency": idem,
|
||||
@@ -371,7 +371,7 @@ def main():
|
||||
"single Docker host",
|
||||
"closed-loop coordinated omission",
|
||||
"successful-response latency percentiles only",
|
||||
"PostgreSQL and Nginx remain single points of failure",
|
||||
"PostgreSQL and HAProxy remain single points of failure",
|
||||
],
|
||||
}
|
||||
destination = ROOT / args.output
|
||||
|
||||
Executable
+577
@@ -0,0 +1,577 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure the price of horizontal API scaling for lab 3 grade 5."""
|
||||
|
||||
import argparse
|
||||
from collections import Counter, defaultdict
|
||||
import datetime as dt
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure # noqa: E402
|
||||
|
||||
|
||||
def command(args, env=None, timeout=240, check=True):
|
||||
result = subprocess.run(
|
||||
[str(value) for value in args],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
if check and result.returncode:
|
||||
detail = (result.stderr or result.stdout).strip().splitlines()
|
||||
suffix = f": {detail[-1]}" if detail else ""
|
||||
raise RuntimeError(
|
||||
f"{' '.join(map(str, args[:4]))} exited {result.returncode}{suffix}"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def compose(env_file, *args, env=None, timeout=240, check=True):
|
||||
return command(
|
||||
["docker", "compose", "--env-file", env_file, *args],
|
||||
env=env,
|
||||
timeout=timeout,
|
||||
check=check,
|
||||
)
|
||||
|
||||
|
||||
def inspect(container_id):
|
||||
raw = command(
|
||||
["docker", "inspect", "--format", "{{json .}}", container_id]
|
||||
).stdout
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def service_ids(env_file, service=None):
|
||||
args = ["ps", "-q"]
|
||||
if service:
|
||||
args.append(service)
|
||||
return [line for line in compose(env_file, *args).stdout.splitlines() if line]
|
||||
|
||||
|
||||
def wait_healthy(env_file, service, expected=1, timeout=120):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
healthy = []
|
||||
for container_id in service_ids(env_file, service):
|
||||
state = inspect(container_id)["State"]
|
||||
if state.get("Health", {}).get("Status") == "healthy":
|
||||
healthy.append(container_id)
|
||||
if len(healthy) == expected:
|
||||
return healthy
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError(f"{service}: expected {expected} healthy containers")
|
||||
|
||||
|
||||
def percentile(values, ratio):
|
||||
if not values:
|
||||
return None
|
||||
ordered = sorted(values)
|
||||
return round(ordered[max(0, math.ceil(ratio * len(ordered)) - 1)], 3)
|
||||
|
||||
|
||||
def memory_mib(raw):
|
||||
value = raw.split("/", 1)[0].strip()
|
||||
units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
|
||||
for suffix in ("GiB", "MiB", "KiB", "B"):
|
||||
if value.endswith(suffix):
|
||||
return round(float(value[:-len(suffix)]) * units[suffix] / 1024**2, 3)
|
||||
return 0.0
|
||||
|
||||
|
||||
def profile_environment(profile):
|
||||
env = dict(os.environ)
|
||||
env.update({
|
||||
"API_REPLICAS": str(profile["replicas"]),
|
||||
"API_CPU_LIMIT": f"{profile['api_cpu']:.2f}",
|
||||
"API_MEMORY_LIMIT": profile["api_memory"],
|
||||
"DB_MAX_CONNS": str(profile["db_max_conns"]),
|
||||
"DB_MIN_CONNS": str(profile["db_min_conns"]),
|
||||
"POSTGRES_CPU_LIMIT": f"{profile['postgres_cpu']:.2f}",
|
||||
"POSTGRES_MEMORY_LIMIT": profile["postgres_memory"],
|
||||
})
|
||||
return env
|
||||
|
||||
|
||||
def configure_profile(env_file, profile):
|
||||
env = profile_environment(profile)
|
||||
compose(
|
||||
env_file,
|
||||
"up", "-d", "--no-deps", "postgres",
|
||||
env=env,
|
||||
timeout=180,
|
||||
)
|
||||
wait_healthy(env_file, "postgres")
|
||||
compose(
|
||||
env_file,
|
||||
"up", "-d", "--no-deps",
|
||||
"--scale", f"api={profile['replicas']}",
|
||||
"--force-recreate", "api",
|
||||
env=env,
|
||||
timeout=240,
|
||||
)
|
||||
api = wait_healthy(env_file, "api", profile["replicas"])
|
||||
wait_healthy(env_file, "load-balancer")
|
||||
time.sleep(1.5)
|
||||
|
||||
expected_cpu = int(profile["api_cpu"] * 1_000_000_000)
|
||||
expected_memory = parse_memory_limit(profile["api_memory"])
|
||||
applied = []
|
||||
for container_id in api:
|
||||
data = inspect(container_id)
|
||||
actual_env = dict(item.split("=", 1) for item in data["Config"]["Env"] if "=" in item)
|
||||
host = data["HostConfig"]
|
||||
if host.get("NanoCpus") != expected_cpu or host.get("Memory") != expected_memory:
|
||||
raise RuntimeError("API container limits do not match the selected profile")
|
||||
if actual_env.get("DB_MAX_CONNS") != str(profile["db_max_conns"]):
|
||||
raise RuntimeError("API DB pool size does not match the selected profile")
|
||||
applied.append({
|
||||
"instance": data["Config"]["Hostname"],
|
||||
"cpu_limit": host.get("NanoCpus", 0) / 1_000_000_000,
|
||||
"memory_limit_mib": round(host.get("Memory", 0) / 1024**2, 1),
|
||||
"db_max_conns": int(actual_env["DB_MAX_CONNS"]),
|
||||
})
|
||||
|
||||
postgres = inspect(service_ids(env_file, "postgres")[0])
|
||||
expected_pg_cpu = int(profile["postgres_cpu"] * 1_000_000_000)
|
||||
if postgres["HostConfig"].get("NanoCpus") != expected_pg_cpu:
|
||||
raise RuntimeError("PostgreSQL CPU limit does not match the selected profile")
|
||||
return {
|
||||
"api": sorted(applied, key=lambda item: item["instance"]),
|
||||
"postgres_cpu_limit": postgres["HostConfig"].get("NanoCpus", 0) / 1_000_000_000,
|
||||
"postgres_memory_limit_mib": round(postgres["HostConfig"].get("Memory", 0) / 1024**2, 1),
|
||||
}
|
||||
|
||||
|
||||
def parse_memory_limit(value):
|
||||
suffix = value[-1].upper()
|
||||
multiplier = {"K": 1024, "M": 1024**2, "G": 1024**3}[suffix]
|
||||
return int(value[:-1]) * multiplier
|
||||
|
||||
|
||||
def resource_targets(env_file):
|
||||
targets = {}
|
||||
for container_id in service_ids(env_file):
|
||||
data = inspect(container_id)
|
||||
service = data["Config"]["Labels"].get("com.docker.compose.service", "other")
|
||||
if service in {"api", "postgres", "redis", "load-balancer", "frontend"}:
|
||||
targets[container_id] = {
|
||||
"service": service,
|
||||
"name": data["Name"].lstrip("/"),
|
||||
}
|
||||
return targets
|
||||
|
||||
|
||||
def stats_snapshot(targets, elapsed):
|
||||
if not targets:
|
||||
return {"t": round(elapsed, 3), "services": {}}
|
||||
result = command(
|
||||
["docker", "stats", "--no-stream", "--format", "{{json .}}", *targets],
|
||||
timeout=15,
|
||||
check=False,
|
||||
)
|
||||
grouped = defaultdict(lambda: {"cpu_percent": 0.0, "memory_mib": 0.0, "containers": 0})
|
||||
for line in result.stdout.splitlines():
|
||||
try:
|
||||
row = json.loads(line)
|
||||
container_id = row.get("ID", "")
|
||||
match = next((key for key in targets if key.startswith(container_id)), None)
|
||||
if not match:
|
||||
continue
|
||||
values = grouped[targets[match]["service"]]
|
||||
values["cpu_percent"] += float(row.get("CPUPerc", "0%").rstrip("%"))
|
||||
values["memory_mib"] += memory_mib(row.get("MemUsage", "0B / 0B"))
|
||||
values["containers"] += 1
|
||||
except (ValueError, json.JSONDecodeError):
|
||||
continue
|
||||
return {
|
||||
"t": round(elapsed, 3),
|
||||
"services": {
|
||||
name: {
|
||||
"cpu_percent": round(values["cpu_percent"], 3),
|
||||
"memory_mib": round(values["memory_mib"], 3),
|
||||
"containers": values["containers"],
|
||||
}
|
||||
for name, values in sorted(grouped.items())
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def summarize_resources(snapshots):
|
||||
grouped = defaultdict(lambda: {"cpu": [], "memory": [], "containers": []})
|
||||
for snapshot in snapshots:
|
||||
for service, values in snapshot["services"].items():
|
||||
grouped[service]["cpu"].append(values["cpu_percent"])
|
||||
grouped[service]["memory"].append(values["memory_mib"])
|
||||
grouped[service]["containers"].append(values["containers"])
|
||||
return {
|
||||
service: {
|
||||
"cpu_percent_avg": round(sum(values["cpu"]) / len(values["cpu"]), 3),
|
||||
"cpu_percent_max": round(max(values["cpu"]), 3),
|
||||
"memory_mib_avg": round(sum(values["memory"]) / len(values["memory"]), 3),
|
||||
"memory_mib_max": round(max(values["memory"]), 3),
|
||||
"containers": max(values["containers"]),
|
||||
}
|
||||
for service, values in sorted(grouped.items())
|
||||
if values["cpu"]
|
||||
}
|
||||
|
||||
|
||||
def run_phase(name, kind, actor, chat, base_url, duration, concurrency, env_file, profile):
|
||||
records = []
|
||||
snapshots = []
|
||||
lock = threading.Lock()
|
||||
stop = threading.Event()
|
||||
barrier = threading.Barrier(concurrency + 1)
|
||||
targets = resource_targets(env_file)
|
||||
started = time.monotonic()
|
||||
|
||||
def worker(worker_id):
|
||||
client = Client(base_url, timeout=5)
|
||||
index = 0
|
||||
barrier.wait()
|
||||
while not stop.is_set():
|
||||
request_started = time.monotonic()
|
||||
try:
|
||||
response = client.message(
|
||||
actor,
|
||||
chat,
|
||||
f"grade5-{name}-{worker_id}-{index}",
|
||||
)
|
||||
row = {
|
||||
"t": round(time.monotonic() - started, 3),
|
||||
"ok": True,
|
||||
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
|
||||
"instance": response["headers"].get("X-Instance-Id", ""),
|
||||
}
|
||||
except Failure as exc:
|
||||
row = {
|
||||
"t": round(time.monotonic() - started, 3),
|
||||
"ok": False,
|
||||
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
|
||||
"error": str(exc),
|
||||
}
|
||||
with lock:
|
||||
records.append(row)
|
||||
index += 1
|
||||
|
||||
def sampler():
|
||||
while not stop.is_set():
|
||||
snapshots.append(stats_snapshot(targets, time.monotonic() - started))
|
||||
stop.wait(1)
|
||||
|
||||
workers = [
|
||||
threading.Thread(target=worker, args=(index,), daemon=True)
|
||||
for index in range(concurrency)
|
||||
]
|
||||
for thread in workers:
|
||||
thread.start()
|
||||
stats_thread = threading.Thread(target=sampler, daemon=True)
|
||||
stats_thread.start()
|
||||
started = time.monotonic()
|
||||
barrier.wait()
|
||||
time.sleep(duration)
|
||||
stop.set()
|
||||
for thread in workers:
|
||||
thread.join(10)
|
||||
stats_thread.join(15)
|
||||
|
||||
elapsed = time.monotonic() - started
|
||||
successes = [row for row in records if row["ok"]]
|
||||
failures = [row for row in records if not row["ok"]]
|
||||
latencies = [row["latency_ms"] for row in successes]
|
||||
buckets = defaultdict(lambda: {"successful": 0, "failed": 0})
|
||||
for row in records:
|
||||
key = str(max(0, int(row["t"])))
|
||||
buckets[key]["successful" if row["ok"] else "failed"] += 1
|
||||
total = len(records)
|
||||
return {
|
||||
"name": name,
|
||||
"kind": kind,
|
||||
"profile": profile["name"],
|
||||
"replicas": profile["replicas"],
|
||||
"concurrency": concurrency,
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"successful_requests": len(successes),
|
||||
"failed_requests": len(failures),
|
||||
"error_ratio": round(len(failures) / total, 6) if total else None,
|
||||
"errors": dict(sorted(Counter(row["error"] for row in failures).items())),
|
||||
"successful_rps": round(len(successes) / elapsed, 3),
|
||||
"successful_latency_ms": {
|
||||
"p50": percentile(latencies, 0.50),
|
||||
"p95": percentile(latencies, 0.95),
|
||||
"p99": percentile(latencies, 0.99),
|
||||
},
|
||||
"instances": sorted({row["instance"] for row in successes if row.get("instance")}),
|
||||
"per_second": dict(sorted(buckets.items(), key=lambda item: int(item[0]))),
|
||||
"resource_samples": snapshots,
|
||||
"resources": summarize_resources(snapshots),
|
||||
}
|
||||
|
||||
|
||||
def dataset(env_file):
|
||||
sql = (
|
||||
"select count(*)::text || '|' || pg_database_size(current_database())::text "
|
||||
"from messages"
|
||||
)
|
||||
result = compose(
|
||||
env_file,
|
||||
"exec", "-T", "postgres", "sh", "-c",
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"',
|
||||
"sh", sql,
|
||||
)
|
||||
count, size = result.stdout.strip().split("|", 1)
|
||||
return {"messages": int(count), "database_size_bytes": int(size)}
|
||||
|
||||
|
||||
def host_environment(env_file):
|
||||
mem_total_kib = 0
|
||||
with open("/proc/meminfo", encoding="utf-8") as source:
|
||||
for line in source:
|
||||
if line.startswith("MemTotal:"):
|
||||
mem_total_kib = int(line.split()[1])
|
||||
break
|
||||
images = compose(env_file, "config", "--images").stdout.splitlines()
|
||||
return {
|
||||
"utc_started_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"git_commit": command(["git", "rev-parse", "HEAD"]).stdout.strip(),
|
||||
"working_tree_dirty": bool(command(["git", "status", "--porcelain"]).stdout.strip()),
|
||||
"logical_cpu_count": os.cpu_count(),
|
||||
"host_memory_mib": round(mem_total_kib / 1024, 1),
|
||||
"kernel": platform.release(),
|
||||
"python": platform.python_version(),
|
||||
"docker_engine": command(["docker", "version", "--format", "{{.Server.Version}}"]).stdout.strip(),
|
||||
"docker_compose": command(["docker", "compose", "version", "--short"]).stdout.strip(),
|
||||
"images": images,
|
||||
}
|
||||
|
||||
|
||||
def analyze(phases):
|
||||
sweep = sorted(
|
||||
(phase for phase in phases if phase["kind"] == "saturation"),
|
||||
key=lambda phase: phase["concurrency"],
|
||||
)
|
||||
saturation = None
|
||||
previous = None
|
||||
for phase in sweep:
|
||||
if previous:
|
||||
gain = (phase["successful_rps"] - previous["successful_rps"]) / max(previous["successful_rps"], 0.001)
|
||||
latency_growth = phase["successful_latency_ms"]["p95"] / max(previous["successful_latency_ms"]["p95"], 0.001)
|
||||
if saturation is None and gain < 0.10 and latency_growth >= 1.50:
|
||||
saturation = {
|
||||
"concurrency": phase["concurrency"],
|
||||
"previous_concurrency": previous["concurrency"],
|
||||
"rps_gain_percent": round(gain * 100, 2),
|
||||
"p95_growth_ratio": round(latency_growth, 3),
|
||||
}
|
||||
previous = phase
|
||||
|
||||
baseline = next(phase for phase in phases if phase["name"] == "growth-r4")
|
||||
improved = next(phase for phase in phases if phase["name"] == "improved-db-cpu-r4")
|
||||
rps_change = (improved["successful_rps"] - baseline["successful_rps"]) / max(baseline["successful_rps"], 0.001) * 100
|
||||
p95_change = (improved["successful_latency_ms"]["p95"] - baseline["successful_latency_ms"]["p95"]) / max(baseline["successful_latency_ms"]["p95"], 0.001) * 100
|
||||
confirmed = (
|
||||
rps_change >= 5
|
||||
and improved["error_ratio"] <= baseline["error_ratio"] + 0.001
|
||||
) or p95_change <= -10
|
||||
|
||||
resources = baseline["resources"]
|
||||
pg = resources.get("postgres", {})
|
||||
lb = resources.get("load-balancer", {})
|
||||
bottleneck = "undetermined"
|
||||
if pg.get("cpu_percent_avg", 0) >= 70 and pg.get("cpu_percent_max", 0) >= 95:
|
||||
bottleneck = "postgres_cpu"
|
||||
elif lb.get("cpu_percent_avg", 0) >= 20 and lb.get("cpu_percent_max", 0) >= 24:
|
||||
bottleneck = "load_balancer_cpu"
|
||||
improved_resources = improved["resources"]
|
||||
improved_lb = improved_resources.get("load-balancer", {})
|
||||
remaining_bottleneck = "undetermined"
|
||||
if improved_lb.get("cpu_percent_avg", 0) >= 20 and improved_lb.get("cpu_percent_max", 0) >= 24:
|
||||
remaining_bottleneck = "load_balancer_cpu"
|
||||
return {
|
||||
"saturation_point": saturation,
|
||||
"observed_bottleneck": bottleneck,
|
||||
"baseline_resource_snapshot": resources,
|
||||
"improved_resource_snapshot": improved_resources,
|
||||
"remaining_bottleneck_after_improvement": remaining_bottleneck,
|
||||
"improvement": {
|
||||
"change": "PostgreSQL CPU limit: 1.00 -> 2.00; API profile unchanged",
|
||||
"baseline_phase": baseline["name"],
|
||||
"improved_phase": improved["name"],
|
||||
"rps_change_percent": round(rps_change, 2),
|
||||
"p95_change_percent": round(p95_change, 2),
|
||||
"confirmed": confirmed,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def profiles():
|
||||
base = {"postgres_cpu": 1.0, "postgres_memory": "512M"}
|
||||
return {
|
||||
"fixed-r1": {**base, "name": "fixed-r1", "replicas": 1, "api_cpu": 1.0, "api_memory": "512M", "db_max_conns": 16, "db_min_conns": 2},
|
||||
"fixed-r2": {**base, "name": "fixed-r2", "replicas": 2, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
|
||||
"fixed-r4": {**base, "name": "fixed-r4", "replicas": 4, "api_cpu": 0.25, "api_memory": "128M", "db_max_conns": 4, "db_min_conns": 1},
|
||||
"growth-r1": {**base, "name": "growth-r1", "replicas": 1, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
|
||||
"growth-r2": {**base, "name": "growth-r2", "replicas": 2, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
|
||||
"growth-r4": {**base, "name": "growth-r4", "replicas": 4, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
|
||||
"improved-db-cpu-r4": {**base, "name": "improved-db-cpu-r4", "replicas": 4, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1, "postgres_cpu": 2.0},
|
||||
}
|
||||
|
||||
|
||||
def run_warmup(actor, chat, args, profile):
|
||||
phase = run_phase(
|
||||
f"warmup-{profile['name']}",
|
||||
"warmup",
|
||||
actor,
|
||||
chat,
|
||||
args.base_url,
|
||||
args.warmup_seconds,
|
||||
min(4, args.comparison_concurrency),
|
||||
args.env_file,
|
||||
profile,
|
||||
)
|
||||
if phase["failed_requests"]:
|
||||
raise RuntimeError(f"warmup failed for {profile['name']}")
|
||||
return phase["successful_requests"]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://localhost:8080")
|
||||
parser.add_argument("--env-file", default=".env")
|
||||
parser.add_argument("--duration", type=float, default=10)
|
||||
parser.add_argument("--warmup-seconds", type=float, default=2)
|
||||
parser.add_argument("--comparison-concurrency", type=int, default=32)
|
||||
parser.add_argument("--sweep", default="1,2,4,8,16,32")
|
||||
parser.add_argument("--output", default="evidence/grade5/results.json")
|
||||
args = parser.parse_args()
|
||||
sweep = [int(value) for value in args.sweep.split(",")]
|
||||
if args.duration <= 0 or args.warmup_seconds <= 0:
|
||||
raise SystemExit("durations must be positive")
|
||||
if not sweep or min(sweep) < 1 or max(sweep + [args.comparison_concurrency]) > 64:
|
||||
raise SystemExit("concurrency must be 1..64")
|
||||
|
||||
all_profiles = profiles()
|
||||
restore = {
|
||||
"name": "restore-default",
|
||||
"replicas": 2,
|
||||
"api_cpu": 0.5,
|
||||
"api_memory": "256M",
|
||||
"db_max_conns": 10,
|
||||
"db_min_conns": 2,
|
||||
"postgres_cpu": 1.0,
|
||||
"postgres_memory": "512M",
|
||||
}
|
||||
phases = []
|
||||
applied = {}
|
||||
warmup_requests = 0
|
||||
restoration = {"attempted": False, "succeeded": False}
|
||||
environment = host_environment(args.env_file)
|
||||
start_dataset = dataset(args.env_file)
|
||||
try:
|
||||
client = Client(args.base_url, timeout=8)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
|
||||
saturation_profile = all_profiles["growth-r2"]
|
||||
applied[saturation_profile["name"]] = configure_profile(args.env_file, saturation_profile)
|
||||
warmup_requests += run_warmup(alice, chat, args, saturation_profile)
|
||||
for concurrency in sweep:
|
||||
print(f"saturation: concurrency={concurrency}", flush=True)
|
||||
phases.append(run_phase(
|
||||
f"saturation-c{concurrency}", "saturation", alice, chat,
|
||||
args.base_url, args.duration, concurrency, args.env_file,
|
||||
saturation_profile,
|
||||
))
|
||||
|
||||
for profile_name in (
|
||||
"fixed-r1", "fixed-r2", "fixed-r4",
|
||||
"growth-r1", "growth-r2", "growth-r4",
|
||||
"improved-db-cpu-r4",
|
||||
):
|
||||
profile = all_profiles[profile_name]
|
||||
print(f"profile: {profile_name}", flush=True)
|
||||
applied[profile_name] = configure_profile(args.env_file, profile)
|
||||
warmup_requests += run_warmup(alice, chat, args, profile)
|
||||
kind = "improvement" if profile_name.startswith("improved") else profile_name.split("-", 1)[0]
|
||||
phases.append(run_phase(
|
||||
profile_name, kind, alice, chat, args.base_url,
|
||||
args.duration, args.comparison_concurrency, args.env_file,
|
||||
profile,
|
||||
))
|
||||
end_dataset = dataset(args.env_file)
|
||||
finally:
|
||||
restoration["attempted"] = True
|
||||
try:
|
||||
restoration["applied"] = configure_profile(args.env_file, restore)
|
||||
restoration["succeeded"] = True
|
||||
except Exception as exc:
|
||||
restoration["error"] = f"{type(exc).__name__}: {exc}"
|
||||
|
||||
if not restoration["succeeded"]:
|
||||
raise RuntimeError("experiment completed but default Compose profile was not restored")
|
||||
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"environment": environment,
|
||||
"hypothesis": (
|
||||
"API replicas stop improving throughput when shared PostgreSQL CPU saturates; "
|
||||
"adding PostgreSQL CPU should improve the unchanged four-replica profile."
|
||||
),
|
||||
"methodology": {
|
||||
"workload": "closed-loop POST messages through the public HAProxy gateway; unique idempotency keys; no client retries",
|
||||
"duration_seconds_per_phase": args.duration,
|
||||
"warmup_seconds_per_profile": args.warmup_seconds,
|
||||
"comparison_concurrency": args.comparison_concurrency,
|
||||
"saturation_concurrency": sweep,
|
||||
"fixed_total_budget": "1.00 API CPU, 512 MiB API RAM, 16 total DB pool connections",
|
||||
"growth_budget": "0.50 API CPU, 256 MiB RAM and 8 DB pool connections per replica",
|
||||
"improvement": "only PostgreSQL CPU changes from 1.00 to 2.00",
|
||||
"saturation_rule": "RPS gain below 10% and p95 growth at least 1.5x versus the previous step",
|
||||
"cpu_bottleneck_rule": "average at least 70% and peak at least 95% of PostgreSQL CPU limit; service-specific threshold for the 0.25 CPU load balancer",
|
||||
},
|
||||
"profiles": all_profiles,
|
||||
"applied_limits": applied,
|
||||
"phases": phases,
|
||||
"analysis": analyze(phases),
|
||||
"dataset": {
|
||||
"before": start_dataset,
|
||||
"after": end_dataset,
|
||||
"warmup_successful_requests": warmup_requests,
|
||||
},
|
||||
"restoration": restoration,
|
||||
"limitations": [
|
||||
"single Docker host: replicas do not prove host-level availability",
|
||||
"closed-loop workload has coordinated omission",
|
||||
"successful-response latency percentiles are reported separately from failures",
|
||||
"one measured run per profile; host scheduling noise is possible",
|
||||
],
|
||||
}
|
||||
destination = ROOT / args.output
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
try:
|
||||
displayed_output = destination.relative_to(ROOT)
|
||||
except ValueError:
|
||||
displayed_output = destination
|
||||
print(json.dumps({
|
||||
"output": str(displayed_output),
|
||||
"analysis": report["analysis"],
|
||||
"restoration": restoration["succeeded"],
|
||||
}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -148,7 +148,7 @@ def summary(report, output):
|
||||
"## Method", "",
|
||||
f"- concurrency: {method['concurrency']};",
|
||||
f"- baseline duration: {method['baseline_seconds']} s;",
|
||||
"- load: closed-loop POST messages through Nginx;",
|
||||
"- load: closed-loop POST messages through the public HAProxy gateway;",
|
||||
"- failover: both original API containers stopped and restored in turn;",
|
||||
"- rolling update: one surge replica, old replicas replaced one by one;",
|
||||
"- percentiles cover successful responses; failures are separate.", "",
|
||||
@@ -178,7 +178,7 @@ def summary(report, output):
|
||||
"", "",
|
||||
"## Scope", "",
|
||||
"Single-host process failure and rolling replacement only. PostgreSQL "
|
||||
"and Nginx remain single points of failure.", "",
|
||||
"and HAProxy remain single points of failure.", "",
|
||||
]
|
||||
output.write_text("\n".join(lines))
|
||||
|
||||
|
||||
Executable
+322
@@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render grade-5 scaling evidence as SVG, CSV and Markdown."""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import html
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
STYLE = """
|
||||
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
.title { font-size: 26px; font-weight: 650; }
|
||||
.subtitle { font-size: 14px; fill: #aeb2b8; }
|
||||
.label { font-size: 14px; fill: #c8cbd0; }
|
||||
.small { font-size: 12px; fill: #a2a5aa; }
|
||||
.grid { stroke: #34373d; stroke-width: 1; }
|
||||
.axis { stroke: #747981; stroke-width: 1.5; }
|
||||
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
|
||||
.line2 { fill: none; stroke: #92979f; stroke-width: 3; }
|
||||
.point { fill: #f4f5f6; }
|
||||
.point2 { fill: #92979f; }
|
||||
.bar { fill: #d7d9dc; }
|
||||
.bar2 { fill: #777c84; }
|
||||
.bar3 { fill: #4c5057; }
|
||||
.threshold { stroke: #8f949c; stroke-width: 2; stroke-dasharray: 7 5; }
|
||||
"""
|
||||
|
||||
|
||||
def svg(title, subtitle, body, width=1120, height=620):
|
||||
return f"""<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" viewBox="0 0 {width} {height}">
|
||||
<rect width="100%" height="100%" fill="#0e1013"/>
|
||||
<style>{STYLE}</style>
|
||||
<text x="50" y="44" class="title">{html.escape(title)}</text>
|
||||
<text x="50" y="70" class="subtitle">{html.escape(subtitle)}</text>
|
||||
{body}
|
||||
</svg>
|
||||
"""
|
||||
|
||||
|
||||
def grid(maximum, left, top, width, height, suffix=""):
|
||||
maximum = maximum or 1
|
||||
parts = []
|
||||
for step in range(6):
|
||||
value = maximum * step / 5
|
||||
y = top + height - height * step / 5
|
||||
parts.append(f'<line x1="{left}" y1="{y:.1f}" x2="{left+width}" y2="{y:.1f}" class="grid"/>')
|
||||
parts.append(f'<text x="{left-12}" y="{y+4:.1f}" text-anchor="end" class="small">{value:.0f}{suffix}</text>')
|
||||
return parts
|
||||
|
||||
|
||||
def phase_map(report):
|
||||
return {phase["name"]: phase for phase in report["phases"]}
|
||||
|
||||
|
||||
def saturation_graph(report, output):
|
||||
phases = sorted(
|
||||
(phase for phase in report["phases"] if phase["kind"] == "saturation"),
|
||||
key=lambda phase: phase["concurrency"],
|
||||
)
|
||||
left, top, width, height = 90, 110, 940, 390
|
||||
max_rps = max(phase["successful_rps"] for phase in phases) * 1.1
|
||||
max_p95 = max(phase["successful_latency_ms"]["p95"] for phase in phases) * 1.1
|
||||
parts = grid(max_rps, left, top, width, height)
|
||||
points_rps, points_p95 = [], []
|
||||
for index, phase in enumerate(phases):
|
||||
x = left + width * index / max(1, len(phases) - 1)
|
||||
y_rps = top + height - height * phase["successful_rps"] / max_rps
|
||||
y_p95 = top + height - height * phase["successful_latency_ms"]["p95"] / max_p95
|
||||
points_rps.append(f"{x:.1f},{y_rps:.1f}")
|
||||
points_p95.append(f"{x:.1f},{y_p95:.1f}")
|
||||
parts.append(f'<circle cx="{x:.1f}" cy="{y_rps:.1f}" r="6" class="point"/>')
|
||||
parts.append(f'<circle cx="{x:.1f}" cy="{y_p95:.1f}" r="6" class="point2"/>')
|
||||
parts.append(f'<text x="{x:.1f}" y="{top+height+28}" text-anchor="middle" class="label">{phase["concurrency"]}</text>')
|
||||
parts.append(f'<text x="{x:.1f}" y="{y_rps-12:.1f}" text-anchor="middle" class="small">{phase["successful_rps"]:.0f}</text>')
|
||||
parts.append(f'<polyline points="{" ".join(points_rps)}" class="line"/>')
|
||||
parts.append(f'<polyline points="{" ".join(points_p95)}" class="line2"/>')
|
||||
parts.append(f'<text x="{left}" y="{top+height+62}" class="label">Concurrency</text>')
|
||||
parts.append(f'<text x="{left+720}" y="{top+height+62}" class="label">white: RPS · gray: p95 normalized</text>')
|
||||
parts.append(f'<text x="30" y="{top+height/2}" transform="rotate(-90 30 {top+height/2})" class="label">Successful requests per second</text>')
|
||||
output.write_text(svg(
|
||||
"Saturation curve",
|
||||
"Two API replicas; 1 CPU total; 16 total DB connections; PostgreSQL limited to 1 CPU",
|
||||
"\n".join(parts),
|
||||
), encoding="utf-8")
|
||||
|
||||
|
||||
def comparison_graph(report, kind, output):
|
||||
phases = sorted(
|
||||
(phase for phase in report["phases"] if phase["kind"] == kind),
|
||||
key=lambda phase: phase["replicas"],
|
||||
)
|
||||
left, top, width, height = 90, 110, 940, 370
|
||||
max_rps = max(phase["successful_rps"] for phase in phases) * 1.15
|
||||
parts = grid(max_rps, left, top, width, height)
|
||||
slot = width / len(phases)
|
||||
for index, phase in enumerate(phases):
|
||||
bar_width = min(150, slot * 0.52)
|
||||
x = left + index * slot + (slot - bar_width) / 2
|
||||
bar_height = height * phase["successful_rps"] / max_rps
|
||||
y = top + height - bar_height
|
||||
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="14" class="bar"/>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-12:.1f}" text-anchor="middle" class="label">{phase["successful_rps"]:.1f} RPS</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+30}" text-anchor="middle" class="label">{phase["replicas"]} replica(s)</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+52}" text-anchor="middle" class="small">p95 {phase["successful_latency_ms"]["p95"]:.1f} ms</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+70}" text-anchor="middle" class="small">errors {phase["error_ratio"]*100:.3f}%</text>')
|
||||
if kind == "fixed":
|
||||
title = "Fixed total API budget"
|
||||
subtitle = "1 CPU · 512 MiB · 16 DB connections split across 1, 2 and 4 replicas"
|
||||
else:
|
||||
title = "Growing API budget"
|
||||
subtitle = "Each replica adds 0.5 CPU · 256 MiB · 8 DB connections; PostgreSQL remains at 1 CPU"
|
||||
output.write_text(svg(title, subtitle, "\n".join(parts)), encoding="utf-8")
|
||||
|
||||
|
||||
def bottleneck_graph(report, output):
|
||||
phases = sorted(
|
||||
(phase for phase in report["phases"] if phase["kind"] == "growth"),
|
||||
key=lambda phase: phase["replicas"],
|
||||
)
|
||||
services = ("api", "postgres", "load-balancer", "frontend", "redis")
|
||||
left, top, width, height = 90, 110, 940, 370
|
||||
maximum = max(
|
||||
phase["resources"].get(service, {}).get("cpu_percent_avg", 0)
|
||||
for phase in phases for service in services
|
||||
) * 1.15 or 1
|
||||
parts = grid(maximum, left, top, width, height)
|
||||
slot = width / len(phases)
|
||||
colors = ("bar", "bar2", "bar3", "bar3", "bar3")
|
||||
for phase_index, phase in enumerate(phases):
|
||||
group_left = left + phase_index * slot
|
||||
bar_width = min(48, slot / 6)
|
||||
for service_index, service in enumerate(services):
|
||||
value = phase["resources"].get(service, {}).get("cpu_percent_avg", 0)
|
||||
x = group_left + slot * 0.12 + service_index * (bar_width + 9)
|
||||
bar_height = height * value / maximum
|
||||
y = top + height - bar_height
|
||||
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="7" class="{colors[service_index]}"/>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-7:.1f}" text-anchor="middle" class="small">{value:.0f}</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+22}" text-anchor="middle" class="small">{service[:3]}</text>')
|
||||
parts.append(f'<text x="{group_left+slot/2:.1f}" y="{top+height+52}" text-anchor="middle" class="label">{phase["replicas"]} replica(s)</text>')
|
||||
parts.append(f'<text x="30" y="{top+height/2}" transform="rotate(-90 30 {top+height/2})" class="label">Average aggregate CPU, %</text>')
|
||||
output.write_text(svg(
|
||||
"Where scaling stops",
|
||||
"Aggregate service CPU during the proportional-growth series (api · postgres · HAProxy · frontend · redis)",
|
||||
"\n".join(parts),
|
||||
), encoding="utf-8")
|
||||
|
||||
|
||||
def improvement_graph(report, output):
|
||||
phases = phase_map(report)
|
||||
baseline = phases["growth-r4"]
|
||||
improved = phases["improved-db-cpu-r4"]
|
||||
pair = [baseline, improved]
|
||||
left, top, panel_width, height = 90, 120, 390, 340
|
||||
parts = []
|
||||
panels = [
|
||||
("Successful RPS", "successful_rps", left),
|
||||
("p95 latency, ms", "p95", 630),
|
||||
]
|
||||
for title, metric, panel_left in panels:
|
||||
values = [
|
||||
phase["successful_rps"] if metric == "successful_rps" else phase["successful_latency_ms"]["p95"]
|
||||
for phase in pair
|
||||
]
|
||||
maximum = max(values) * 1.2 or 1
|
||||
parts.append(f'<text x="{panel_left}" y="{top-22}" class="label">{title}</text>')
|
||||
for index, (phase, value) in enumerate(zip(pair, values)):
|
||||
bar_width = 120
|
||||
x = panel_left + 35 + index * 185
|
||||
bar_height = height * value / maximum
|
||||
y = top + height - bar_height
|
||||
parts.append(f'<rect x="{x}" y="{y:.1f}" width="{bar_width}" height="{bar_height:.1f}" rx="14" class="{"bar" if index else "bar2"}"/>')
|
||||
parts.append(f'<text x="{x+60}" y="{y-10:.1f}" text-anchor="middle" class="label">{value:.1f}</text>')
|
||||
parts.append(f'<text x="{x+60}" y="{top+height+28}" text-anchor="middle" class="small">PG {1 if index == 0 else 2} CPU</text>')
|
||||
analysis = report["analysis"]["improvement"]
|
||||
parts.append(f'<text x="90" y="545" class="label">RPS change: {analysis["rps_change_percent"]:+.2f}% · p95 change: {analysis["p95_change_percent"]:+.2f}% · confirmed: {str(analysis["confirmed"]).lower()}</text>')
|
||||
output.write_text(svg(
|
||||
"Confirmed improvement",
|
||||
"Four API replicas unchanged; only the PostgreSQL CPU limit changes from 1 to 2",
|
||||
"\n".join(parts),
|
||||
), encoding="utf-8")
|
||||
|
||||
|
||||
def write_csv(report, destination):
|
||||
with (destination / "measurements.csv").open("w", newline="", encoding="utf-8") as stream:
|
||||
fields = [
|
||||
"phase", "kind", "replicas", "concurrency", "elapsed_seconds",
|
||||
"successful_requests", "failed_requests", "error_ratio", "successful_rps",
|
||||
"p50_ms", "p95_ms", "p99_ms", "api_cpu_avg", "postgres_cpu_avg",
|
||||
"load_balancer_cpu_avg", "frontend_cpu_avg", "redis_cpu_avg",
|
||||
]
|
||||
writer = csv.DictWriter(stream, fieldnames=fields)
|
||||
writer.writeheader()
|
||||
for phase in report["phases"]:
|
||||
writer.writerow({
|
||||
"phase": phase["name"],
|
||||
"kind": phase["kind"],
|
||||
"replicas": phase["replicas"],
|
||||
"concurrency": phase["concurrency"],
|
||||
"elapsed_seconds": phase["elapsed_seconds"],
|
||||
"successful_requests": phase["successful_requests"],
|
||||
"failed_requests": phase["failed_requests"],
|
||||
"error_ratio": phase["error_ratio"],
|
||||
"successful_rps": phase["successful_rps"],
|
||||
"p50_ms": phase["successful_latency_ms"]["p50"],
|
||||
"p95_ms": phase["successful_latency_ms"]["p95"],
|
||||
"p99_ms": phase["successful_latency_ms"]["p99"],
|
||||
"api_cpu_avg": phase["resources"].get("api", {}).get("cpu_percent_avg"),
|
||||
"postgres_cpu_avg": phase["resources"].get("postgres", {}).get("cpu_percent_avg"),
|
||||
"load_balancer_cpu_avg": phase["resources"].get("load-balancer", {}).get("cpu_percent_avg"),
|
||||
"frontend_cpu_avg": phase["resources"].get("frontend", {}).get("cpu_percent_avg"),
|
||||
"redis_cpu_avg": phase["resources"].get("redis", {}).get("cpu_percent_avg"),
|
||||
})
|
||||
|
||||
with (destination / "timeseries.csv").open("w", newline="", encoding="utf-8") as stream:
|
||||
fields = ["phase", "second", "successful", "failed"]
|
||||
writer = csv.DictWriter(stream, fieldnames=fields)
|
||||
writer.writeheader()
|
||||
for phase in report["phases"]:
|
||||
for second, counts in phase["per_second"].items():
|
||||
writer.writerow({"phase": phase["name"], "second": second, **counts})
|
||||
|
||||
with (destination / "resources.csv").open("w", newline="", encoding="utf-8") as stream:
|
||||
fields = ["phase", "t", "service", "containers", "cpu_percent", "memory_mib"]
|
||||
writer = csv.DictWriter(stream, fieldnames=fields)
|
||||
writer.writeheader()
|
||||
for phase in report["phases"]:
|
||||
for snapshot in phase["resource_samples"]:
|
||||
for service, values in snapshot["services"].items():
|
||||
writer.writerow({
|
||||
"phase": phase["name"], "t": snapshot["t"], "service": service,
|
||||
**values,
|
||||
})
|
||||
|
||||
|
||||
def markdown_summary(report, output):
|
||||
method = report["methodology"]
|
||||
analysis = report["analysis"]
|
||||
improvement = analysis["improvement"]
|
||||
lines = [
|
||||
"# Lab 3 grade-5 scaling experiment", "",
|
||||
f"Generated: {report['environment']['utc_started_at']}.", "",
|
||||
"## Hypothesis", "", report["hypothesis"], "",
|
||||
"## Reproduction", "",
|
||||
f"- workload: {method['workload']};",
|
||||
f"- phase duration: {method['duration_seconds_per_phase']} s;",
|
||||
f"- comparison concurrency: {method['comparison_concurrency']};",
|
||||
f"- saturation sweep: {method['saturation_concurrency']};",
|
||||
f"- fixed budget: {method['fixed_total_budget']};",
|
||||
f"- growth series: {method['growth_budget']};",
|
||||
f"- saturation rule: {method['saturation_rule']};",
|
||||
f"- host: {report['environment']['logical_cpu_count']} logical CPU, {report['environment']['host_memory_mib']} MiB RAM;",
|
||||
f"- Docker {report['environment']['docker_engine']}, Compose {report['environment']['docker_compose']};",
|
||||
"- command: `make grade5-experiment`; warmup runs before every changed profile.", "",
|
||||
"## Measurements", "",
|
||||
"| Phase | Replicas | Concurrency | RPS | Errors | p50 ms | p95 ms | p99 ms | PG CPU avg | API CPU avg |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
]
|
||||
for phase in report["phases"]:
|
||||
latency = phase["successful_latency_ms"]
|
||||
resources = phase["resources"]
|
||||
lines.append(
|
||||
f"| {phase['name']} | {phase['replicas']} | {phase['concurrency']} | "
|
||||
f"{phase['successful_rps']} | {phase['failed_requests']} ({phase['error_ratio']*100:.3f}%) | "
|
||||
f"{latency['p50']} | {latency['p95']} | {latency['p99']} | "
|
||||
f"{resources.get('postgres', {}).get('cpu_percent_avg', 'n/a')} | "
|
||||
f"{resources.get('api', {}).get('cpu_percent_avg', 'n/a')} |"
|
||||
)
|
||||
saturation = analysis["saturation_point"]
|
||||
lines += ["", "## Conclusion", ""]
|
||||
if saturation:
|
||||
lines.append(
|
||||
f"The first saturation signal appears at concurrency {saturation['concurrency']}: "
|
||||
f"RPS changes by {saturation['rps_gain_percent']:+.2f}% while p95 changes by "
|
||||
f"{saturation['p95_growth_ratio']:.2f}x versus concurrency {saturation['previous_concurrency']}."
|
||||
)
|
||||
else:
|
||||
lines.append("The configured sweep did not locate a saturation point by the declared rule.")
|
||||
lines += [
|
||||
"",
|
||||
f"Observed bottleneck classification: `{analysis['observed_bottleneck']}`.", "",
|
||||
f"The PostgreSQL CPU change produced {improvement['rps_change_percent']:+.2f}% RPS and "
|
||||
f"{improvement['p95_change_percent']:+.2f}% p95; confirmed improvement: "
|
||||
f"`{str(improvement['confirmed']).lower()}`.", "",
|
||||
f"After the improvement the remaining bottleneck classification is "
|
||||
f"`{analysis['remaining_bottleneck_after_improvement']}`.", "",
|
||||
"## Graphs", "",
|
||||
"", "",
|
||||
"", "",
|
||||
"", "",
|
||||
"", "",
|
||||
"", "",
|
||||
"## Raw data and limits", "",
|
||||
"Raw aggregates are in `measurements.csv`; one-second request buckets are in "
|
||||
"`timeseries.csv`; every resource sample is in `resources.csv`; complete metadata and "
|
||||
"applied limits are in `results.json`.", "",
|
||||
]
|
||||
lines.extend(f"- {item}" for item in report["limitations"])
|
||||
lines.append("")
|
||||
output.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("results", nargs="?", default="evidence/grade5/results.json")
|
||||
args = parser.parse_args()
|
||||
source = Path(args.results)
|
||||
report = json.loads(source.read_text(encoding="utf-8"))
|
||||
destination = source.parent
|
||||
saturation_graph(report, destination / "01-saturation.svg")
|
||||
comparison_graph(report, "fixed", destination / "02-fixed-total.svg")
|
||||
comparison_graph(report, "growth", destination / "03-resource-growth.svg")
|
||||
bottleneck_graph(report, destination / "04-bottleneck.svg")
|
||||
improvement_graph(report, destination / "05-improvement.svg")
|
||||
write_csv(report, destination)
|
||||
markdown_summary(report, destination / "README.md")
|
||||
print(f"rendered grade-5 evidence in {destination}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+5
-1
@@ -19,6 +19,10 @@ compose up -d --no-deps api
|
||||
compose restart api
|
||||
compose up -d --wait --no-deps api
|
||||
|
||||
# Публичная точка входа перезапускается последней, когда API уже готовы.
|
||||
# Статический frontend возвращается до публичной точки входа.
|
||||
compose restart frontend
|
||||
compose up -d --wait --no-deps frontend
|
||||
|
||||
# Публичная точка входа перезапускается последней, когда API и frontend готовы.
|
||||
compose restart load-balancer
|
||||
compose up -d --wait --no-deps load-balancer
|
||||
|
||||
Executable
+198
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove that the gateway excludes an API replica which loses readiness."""
|
||||
|
||||
import argparse
|
||||
from collections import Counter
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FAULT_IMAGE = "node:24-alpine@sha256:44fce696f38ba8c987c4db89954051000bbd05ffd8d8659ba15ed4a8602e937f"
|
||||
|
||||
|
||||
def command(args, timeout=180, check=True):
|
||||
result = subprocess.run(
|
||||
[str(value) for value in args],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
if check and result.returncode:
|
||||
detail = (result.stderr or result.stdout).strip().splitlines()
|
||||
suffix = f": {detail[-1]}" if detail else ""
|
||||
raise RuntimeError(f"{' '.join(map(str, args[:4]))} failed{suffix}")
|
||||
return result
|
||||
|
||||
|
||||
def compose(env_file, *args):
|
||||
return command(["docker", "compose", "--env-file", env_file, *args])
|
||||
|
||||
|
||||
def inspect(container_id):
|
||||
return json.loads(command([
|
||||
"docker", "inspect", "--format", "{{json .}}", container_id,
|
||||
]).stdout)
|
||||
|
||||
|
||||
def call(host, port, method, path, body=None, headers=None, timeout=5):
|
||||
connection = http.client.HTTPConnection(host, port, timeout=timeout)
|
||||
connection.request(method, path, body=body, headers=headers or {})
|
||||
response = connection.getresponse()
|
||||
payload = response.read()
|
||||
result = {
|
||||
"status": response.status,
|
||||
"instance": response.getheader("X-Instance-Id", ""),
|
||||
"body": payload,
|
||||
}
|
||||
connection.close()
|
||||
return result
|
||||
|
||||
|
||||
def observe_instances(base_host, base_port, requests=16):
|
||||
instances = Counter()
|
||||
statuses = Counter()
|
||||
for _ in range(requests):
|
||||
result = call(base_host, base_port, "GET", "/health/live")
|
||||
statuses[result["status"]] += 1
|
||||
instances[result["instance"]] += 1
|
||||
return statuses, instances
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", default=".env")
|
||||
parser.add_argument("--host", default="localhost")
|
||||
parser.add_argument("--port", type=int, default=8080)
|
||||
parser.add_argument("--output", default="")
|
||||
args = parser.parse_args()
|
||||
|
||||
api_ids = [line for line in compose(args.env_file, "ps", "-q", "api").stdout.splitlines() if line]
|
||||
if len(api_ids) != 2:
|
||||
raise RuntimeError("readiness routing test requires exactly two API replicas")
|
||||
target_id = api_ids[0]
|
||||
target = inspect(target_id)
|
||||
target_instance = target["Config"]["Hostname"]
|
||||
target_ip = next(iter(target["NetworkSettings"]["Networks"].values()))["IPAddress"]
|
||||
|
||||
postgres_id = compose(args.env_file, "ps", "-q", "postgres").stdout.strip()
|
||||
postgres = inspect(postgres_id)
|
||||
postgres_ip = next(iter(postgres["NetworkSettings"]["Networks"].values()))["IPAddress"]
|
||||
|
||||
before_statuses, before_instances = observe_instances(args.host, args.port, 40)
|
||||
if before_statuses != {200: 40} or len(before_instances) != 2:
|
||||
raise RuntimeError("both API replicas must serve traffic before fault injection")
|
||||
|
||||
helper = f"mtusi2026lab3-readiness-fault-{os.getpid()}"
|
||||
rule = [
|
||||
"iptables", "-D", "OUTPUT", "-d", postgres_ip,
|
||||
"-p", "tcp", "--dport", "5432", "-j", "REJECT",
|
||||
"--reject-with", "tcp-reset",
|
||||
]
|
||||
detection_started = time.monotonic()
|
||||
try:
|
||||
script = (
|
||||
"apk add --no-cache iptables >/dev/null && "
|
||||
f"iptables -I OUTPUT -d {postgres_ip} -p tcp --dport 5432 "
|
||||
"-j REJECT --reject-with tcp-reset && "
|
||||
"touch /tmp/fault-ready && sleep 180"
|
||||
)
|
||||
command([
|
||||
"docker", "run", "-d", "--name", helper,
|
||||
"--network", f"container:{target_id}",
|
||||
"--cap-add", "NET_ADMIN", FAULT_IMAGE,
|
||||
"sh", "-c", script,
|
||||
])
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
ready = command(
|
||||
["docker", "exec", helper, "test", "-f", "/tmp/fault-ready"],
|
||||
check=False,
|
||||
)
|
||||
if ready.returncode == 0:
|
||||
break
|
||||
time.sleep(0.2)
|
||||
else:
|
||||
raise RuntimeError("fault helper did not become ready")
|
||||
|
||||
direct = call(target_ip, 8080, "GET", "/health/ready")
|
||||
if direct["status"] != 503:
|
||||
raise RuntimeError(f"target readiness is {direct['status']}, want 503")
|
||||
|
||||
deadline = time.monotonic() + 10
|
||||
while time.monotonic() < deadline:
|
||||
statuses, instances = observe_instances(args.host, args.port, 16)
|
||||
if statuses == {200: 16} and target_instance not in instances:
|
||||
break
|
||||
time.sleep(0.25)
|
||||
else:
|
||||
raise RuntimeError("gateway did not exclude the unready API within 10 seconds")
|
||||
detection_seconds = time.monotonic() - detection_started
|
||||
|
||||
health_statuses, health_instances = observe_instances(args.host, args.port, 40)
|
||||
registration_statuses = Counter()
|
||||
registration_instances = Counter()
|
||||
for _ in range(20):
|
||||
body = json.dumps({
|
||||
"username": "readiness_" + uuid.uuid4().hex[:16],
|
||||
"display_name": "Readiness audit",
|
||||
"password": "T3st-" + uuid.uuid4().hex,
|
||||
}).encode()
|
||||
response = call(
|
||||
args.host,
|
||||
args.port,
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
body=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
timeout=8,
|
||||
)
|
||||
registration_statuses[response["status"]] += 1
|
||||
registration_instances[response["instance"]] += 1
|
||||
|
||||
if health_statuses != {200: 40} or target_instance in health_instances:
|
||||
raise RuntimeError("unready API received liveness traffic through gateway")
|
||||
if registration_statuses != {201: 20} or target_instance in registration_instances:
|
||||
raise RuntimeError("unready API received registration traffic through gateway")
|
||||
finally:
|
||||
command(["docker", "exec", helper, *rule], check=False)
|
||||
command(["docker", "rm", "-f", helper], check=False)
|
||||
|
||||
deadline = time.monotonic() + 15
|
||||
while time.monotonic() < deadline:
|
||||
direct = call(target_ip, 8080, "GET", "/health/ready")
|
||||
if direct["status"] == 200:
|
||||
_, instances = observe_instances(args.host, args.port, 24)
|
||||
if target_instance in instances:
|
||||
break
|
||||
time.sleep(0.25)
|
||||
else:
|
||||
raise RuntimeError("recovered API did not return to gateway rotation")
|
||||
|
||||
result = {
|
||||
"target_instance": target_instance,
|
||||
"detection_seconds": round(detection_seconds, 3),
|
||||
"health_requests": dict(health_statuses),
|
||||
"registration_requests": dict(registration_statuses),
|
||||
"recovered": True,
|
||||
}
|
||||
rendered = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
output = Path(args.output)
|
||||
if not output.is_absolute():
|
||||
output = ROOT / output
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(rendered + "\n", encoding="utf-8")
|
||||
print(rendered)
|
||||
print("PASS: readiness-aware gateway excluded and restored one API replica")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user