Files
mtusi2026lab3/scripts/grade5_experiment.py
T
2026-10-25 21:46:21 +03:00

578 lines
22 KiB
Python
Executable File

#!/usr/bin/env python3
"""Measure the price of horizontal API scaling for lab 3 grade 5."""
import argparse
from collections import Counter, defaultdict
import datetime as dt
import json
import math
import os
from pathlib import Path
import platform
import subprocess
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
def command(args, env=None, timeout=240, check=True):
result = subprocess.run(
[str(value) for value in args],
cwd=ROOT,
env=env,
text=True,
capture_output=True,
timeout=timeout,
check=False,
)
if check and result.returncode:
detail = (result.stderr or result.stdout).strip().splitlines()
suffix = f": {detail[-1]}" if detail else ""
raise RuntimeError(
f"{' '.join(map(str, args[:4]))} exited {result.returncode}{suffix}"
)
return result
def compose(env_file, *args, env=None, timeout=240, check=True):
return command(
["docker", "compose", "--env-file", env_file, *args],
env=env,
timeout=timeout,
check=check,
)
def inspect(container_id):
raw = command(
["docker", "inspect", "--format", "{{json .}}", container_id]
).stdout
return json.loads(raw)
def service_ids(env_file, service=None):
args = ["ps", "-q"]
if service:
args.append(service)
return [line for line in compose(env_file, *args).stdout.splitlines() if line]
def wait_healthy(env_file, service, expected=1, timeout=120):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
healthy = []
for container_id in service_ids(env_file, service):
state = inspect(container_id)["State"]
if state.get("Health", {}).get("Status") == "healthy":
healthy.append(container_id)
if len(healthy) == expected:
return healthy
time.sleep(0.5)
raise RuntimeError(f"{service}: expected {expected} healthy containers")
def percentile(values, ratio):
if not values:
return None
ordered = sorted(values)
return round(ordered[max(0, math.ceil(ratio * len(ordered)) - 1)], 3)
def memory_mib(raw):
value = raw.split("/", 1)[0].strip()
units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
for suffix in ("GiB", "MiB", "KiB", "B"):
if value.endswith(suffix):
return round(float(value[:-len(suffix)]) * units[suffix] / 1024**2, 3)
return 0.0
def profile_environment(profile):
env = dict(os.environ)
env.update({
"API_REPLICAS": str(profile["replicas"]),
"API_CPU_LIMIT": f"{profile['api_cpu']:.2f}",
"API_MEMORY_LIMIT": profile["api_memory"],
"DB_MAX_CONNS": str(profile["db_max_conns"]),
"DB_MIN_CONNS": str(profile["db_min_conns"]),
"POSTGRES_CPU_LIMIT": f"{profile['postgres_cpu']:.2f}",
"POSTGRES_MEMORY_LIMIT": profile["postgres_memory"],
})
return env
def configure_profile(env_file, profile):
env = profile_environment(profile)
compose(
env_file,
"up", "-d", "--no-deps", "postgres",
env=env,
timeout=180,
)
wait_healthy(env_file, "postgres")
compose(
env_file,
"up", "-d", "--no-deps",
"--scale", f"api={profile['replicas']}",
"--force-recreate", "api",
env=env,
timeout=240,
)
api = wait_healthy(env_file, "api", profile["replicas"])
wait_healthy(env_file, "load-balancer")
time.sleep(1.5)
expected_cpu = int(profile["api_cpu"] * 1_000_000_000)
expected_memory = parse_memory_limit(profile["api_memory"])
applied = []
for container_id in api:
data = inspect(container_id)
actual_env = dict(item.split("=", 1) for item in data["Config"]["Env"] if "=" in item)
host = data["HostConfig"]
if host.get("NanoCpus") != expected_cpu or host.get("Memory") != expected_memory:
raise RuntimeError("API container limits do not match the selected profile")
if actual_env.get("DB_MAX_CONNS") != str(profile["db_max_conns"]):
raise RuntimeError("API DB pool size does not match the selected profile")
applied.append({
"instance": data["Config"]["Hostname"],
"cpu_limit": host.get("NanoCpus", 0) / 1_000_000_000,
"memory_limit_mib": round(host.get("Memory", 0) / 1024**2, 1),
"db_max_conns": int(actual_env["DB_MAX_CONNS"]),
})
postgres = inspect(service_ids(env_file, "postgres")[0])
expected_pg_cpu = int(profile["postgres_cpu"] * 1_000_000_000)
if postgres["HostConfig"].get("NanoCpus") != expected_pg_cpu:
raise RuntimeError("PostgreSQL CPU limit does not match the selected profile")
return {
"api": sorted(applied, key=lambda item: item["instance"]),
"postgres_cpu_limit": postgres["HostConfig"].get("NanoCpus", 0) / 1_000_000_000,
"postgres_memory_limit_mib": round(postgres["HostConfig"].get("Memory", 0) / 1024**2, 1),
}
def parse_memory_limit(value):
suffix = value[-1].upper()
multiplier = {"K": 1024, "M": 1024**2, "G": 1024**3}[suffix]
return int(value[:-1]) * multiplier
def resource_targets(env_file):
targets = {}
for container_id in service_ids(env_file):
data = inspect(container_id)
service = data["Config"]["Labels"].get("com.docker.compose.service", "other")
if service in {"api", "postgres", "redis", "load-balancer", "frontend"}:
targets[container_id] = {
"service": service,
"name": data["Name"].lstrip("/"),
}
return targets
def stats_snapshot(targets, elapsed):
if not targets:
return {"t": round(elapsed, 3), "services": {}}
result = command(
["docker", "stats", "--no-stream", "--format", "{{json .}}", *targets],
timeout=15,
check=False,
)
grouped = defaultdict(lambda: {"cpu_percent": 0.0, "memory_mib": 0.0, "containers": 0})
for line in result.stdout.splitlines():
try:
row = json.loads(line)
container_id = row.get("ID", "")
match = next((key for key in targets if key.startswith(container_id)), None)
if not match:
continue
values = grouped[targets[match]["service"]]
values["cpu_percent"] += float(row.get("CPUPerc", "0%").rstrip("%"))
values["memory_mib"] += memory_mib(row.get("MemUsage", "0B / 0B"))
values["containers"] += 1
except (ValueError, json.JSONDecodeError):
continue
return {
"t": round(elapsed, 3),
"services": {
name: {
"cpu_percent": round(values["cpu_percent"], 3),
"memory_mib": round(values["memory_mib"], 3),
"containers": values["containers"],
}
for name, values in sorted(grouped.items())
},
}
def summarize_resources(snapshots):
grouped = defaultdict(lambda: {"cpu": [], "memory": [], "containers": []})
for snapshot in snapshots:
for service, values in snapshot["services"].items():
grouped[service]["cpu"].append(values["cpu_percent"])
grouped[service]["memory"].append(values["memory_mib"])
grouped[service]["containers"].append(values["containers"])
return {
service: {
"cpu_percent_avg": round(sum(values["cpu"]) / len(values["cpu"]), 3),
"cpu_percent_max": round(max(values["cpu"]), 3),
"memory_mib_avg": round(sum(values["memory"]) / len(values["memory"]), 3),
"memory_mib_max": round(max(values["memory"]), 3),
"containers": max(values["containers"]),
}
for service, values in sorted(grouped.items())
if values["cpu"]
}
def run_phase(name, kind, actor, chat, base_url, duration, concurrency, env_file, profile):
records = []
snapshots = []
lock = threading.Lock()
stop = threading.Event()
barrier = threading.Barrier(concurrency + 1)
targets = resource_targets(env_file)
started = time.monotonic()
def worker(worker_id):
client = Client(base_url, timeout=5)
index = 0
barrier.wait()
while not stop.is_set():
request_started = time.monotonic()
try:
response = client.message(
actor,
chat,
f"grade5-{name}-{worker_id}-{index}",
)
row = {
"t": round(time.monotonic() - started, 3),
"ok": True,
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
"instance": response["headers"].get("X-Instance-Id", ""),
}
except Failure as exc:
row = {
"t": round(time.monotonic() - started, 3),
"ok": False,
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
"error": str(exc),
}
with lock:
records.append(row)
index += 1
def sampler():
while not stop.is_set():
snapshots.append(stats_snapshot(targets, time.monotonic() - started))
stop.wait(1)
workers = [
threading.Thread(target=worker, args=(index,), daemon=True)
for index in range(concurrency)
]
for thread in workers:
thread.start()
stats_thread = threading.Thread(target=sampler, daemon=True)
stats_thread.start()
started = time.monotonic()
barrier.wait()
time.sleep(duration)
stop.set()
for thread in workers:
thread.join(10)
stats_thread.join(15)
elapsed = time.monotonic() - started
successes = [row for row in records if row["ok"]]
failures = [row for row in records if not row["ok"]]
latencies = [row["latency_ms"] for row in successes]
buckets = defaultdict(lambda: {"successful": 0, "failed": 0})
for row in records:
key = str(max(0, int(row["t"])))
buckets[key]["successful" if row["ok"] else "failed"] += 1
total = len(records)
return {
"name": name,
"kind": kind,
"profile": profile["name"],
"replicas": profile["replicas"],
"concurrency": concurrency,
"elapsed_seconds": round(elapsed, 3),
"successful_requests": len(successes),
"failed_requests": len(failures),
"error_ratio": round(len(failures) / total, 6) if total else None,
"errors": dict(sorted(Counter(row["error"] for row in failures).items())),
"successful_rps": round(len(successes) / elapsed, 3),
"successful_latency_ms": {
"p50": percentile(latencies, 0.50),
"p95": percentile(latencies, 0.95),
"p99": percentile(latencies, 0.99),
},
"instances": sorted({row["instance"] for row in successes if row.get("instance")}),
"per_second": dict(sorted(buckets.items(), key=lambda item: int(item[0]))),
"resource_samples": snapshots,
"resources": summarize_resources(snapshots),
}
def dataset(env_file):
sql = (
"select count(*)::text || '|' || pg_database_size(current_database())::text "
"from messages"
)
result = compose(
env_file,
"exec", "-T", "postgres", "sh", "-c",
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"',
"sh", sql,
)
count, size = result.stdout.strip().split("|", 1)
return {"messages": int(count), "database_size_bytes": int(size)}
def host_environment(env_file):
mem_total_kib = 0
with open("/proc/meminfo", encoding="utf-8") as source:
for line in source:
if line.startswith("MemTotal:"):
mem_total_kib = int(line.split()[1])
break
images = compose(env_file, "config", "--images").stdout.splitlines()
return {
"utc_started_at": dt.datetime.now(dt.timezone.utc).isoformat(),
"git_commit": command(["git", "rev-parse", "HEAD"]).stdout.strip(),
"working_tree_dirty": bool(command(["git", "status", "--porcelain"]).stdout.strip()),
"logical_cpu_count": os.cpu_count(),
"host_memory_mib": round(mem_total_kib / 1024, 1),
"kernel": platform.release(),
"python": platform.python_version(),
"docker_engine": command(["docker", "version", "--format", "{{.Server.Version}}"]).stdout.strip(),
"docker_compose": command(["docker", "compose", "version", "--short"]).stdout.strip(),
"images": images,
}
def analyze(phases):
sweep = sorted(
(phase for phase in phases if phase["kind"] == "saturation"),
key=lambda phase: phase["concurrency"],
)
saturation = None
previous = None
for phase in sweep:
if previous:
gain = (phase["successful_rps"] - previous["successful_rps"]) / max(previous["successful_rps"], 0.001)
latency_growth = phase["successful_latency_ms"]["p95"] / max(previous["successful_latency_ms"]["p95"], 0.001)
if saturation is None and gain < 0.10 and latency_growth >= 1.50:
saturation = {
"concurrency": phase["concurrency"],
"previous_concurrency": previous["concurrency"],
"rps_gain_percent": round(gain * 100, 2),
"p95_growth_ratio": round(latency_growth, 3),
}
previous = phase
baseline = next(phase for phase in phases if phase["name"] == "growth-r4")
improved = next(phase for phase in phases if phase["name"] == "improved-db-cpu-r4")
rps_change = (improved["successful_rps"] - baseline["successful_rps"]) / max(baseline["successful_rps"], 0.001) * 100
p95_change = (improved["successful_latency_ms"]["p95"] - baseline["successful_latency_ms"]["p95"]) / max(baseline["successful_latency_ms"]["p95"], 0.001) * 100
confirmed = (
rps_change >= 5
and improved["error_ratio"] <= baseline["error_ratio"] + 0.001
) or p95_change <= -10
resources = baseline["resources"]
pg = resources.get("postgres", {})
lb = resources.get("load-balancer", {})
bottleneck = "undetermined"
if pg.get("cpu_percent_avg", 0) >= 70 and pg.get("cpu_percent_max", 0) >= 95:
bottleneck = "postgres_cpu"
elif lb.get("cpu_percent_avg", 0) >= 20 and lb.get("cpu_percent_max", 0) >= 24:
bottleneck = "load_balancer_cpu"
improved_resources = improved["resources"]
improved_lb = improved_resources.get("load-balancer", {})
remaining_bottleneck = "undetermined"
if improved_lb.get("cpu_percent_avg", 0) >= 20 and improved_lb.get("cpu_percent_max", 0) >= 24:
remaining_bottleneck = "load_balancer_cpu"
return {
"saturation_point": saturation,
"observed_bottleneck": bottleneck,
"baseline_resource_snapshot": resources,
"improved_resource_snapshot": improved_resources,
"remaining_bottleneck_after_improvement": remaining_bottleneck,
"improvement": {
"change": "PostgreSQL CPU limit: 1.00 -> 2.00; API profile unchanged",
"baseline_phase": baseline["name"],
"improved_phase": improved["name"],
"rps_change_percent": round(rps_change, 2),
"p95_change_percent": round(p95_change, 2),
"confirmed": confirmed,
},
}
def profiles():
base = {"postgres_cpu": 1.0, "postgres_memory": "512M"}
return {
"fixed-r1": {**base, "name": "fixed-r1", "replicas": 1, "api_cpu": 1.0, "api_memory": "512M", "db_max_conns": 16, "db_min_conns": 2},
"fixed-r2": {**base, "name": "fixed-r2", "replicas": 2, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
"fixed-r4": {**base, "name": "fixed-r4", "replicas": 4, "api_cpu": 0.25, "api_memory": "128M", "db_max_conns": 4, "db_min_conns": 1},
"growth-r1": {**base, "name": "growth-r1", "replicas": 1, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
"growth-r2": {**base, "name": "growth-r2", "replicas": 2, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
"growth-r4": {**base, "name": "growth-r4", "replicas": 4, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
"improved-db-cpu-r4": {**base, "name": "improved-db-cpu-r4", "replicas": 4, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1, "postgres_cpu": 2.0},
}
def run_warmup(actor, chat, args, profile):
phase = run_phase(
f"warmup-{profile['name']}",
"warmup",
actor,
chat,
args.base_url,
args.warmup_seconds,
min(4, args.comparison_concurrency),
args.env_file,
profile,
)
if phase["failed_requests"]:
raise RuntimeError(f"warmup failed for {profile['name']}")
return phase["successful_requests"]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--env-file", default=".env")
parser.add_argument("--duration", type=float, default=10)
parser.add_argument("--warmup-seconds", type=float, default=2)
parser.add_argument("--comparison-concurrency", type=int, default=32)
parser.add_argument("--sweep", default="1,2,4,8,16,32")
parser.add_argument("--output", default="evidence/grade5/results.json")
args = parser.parse_args()
sweep = [int(value) for value in args.sweep.split(",")]
if args.duration <= 0 or args.warmup_seconds <= 0:
raise SystemExit("durations must be positive")
if not sweep or min(sweep) < 1 or max(sweep + [args.comparison_concurrency]) > 64:
raise SystemExit("concurrency must be 1..64")
all_profiles = profiles()
restore = {
"name": "restore-default",
"replicas": 2,
"api_cpu": 0.5,
"api_memory": "256M",
"db_max_conns": 10,
"db_min_conns": 2,
"postgres_cpu": 1.0,
"postgres_memory": "512M",
}
phases = []
applied = {}
warmup_requests = 0
restoration = {"attempted": False, "succeeded": False}
environment = host_environment(args.env_file)
start_dataset = dataset(args.env_file)
try:
client = Client(args.base_url, timeout=8)
alice, _, _, chat = client.bootstrap()
saturation_profile = all_profiles["growth-r2"]
applied[saturation_profile["name"]] = configure_profile(args.env_file, saturation_profile)
warmup_requests += run_warmup(alice, chat, args, saturation_profile)
for concurrency in sweep:
print(f"saturation: concurrency={concurrency}", flush=True)
phases.append(run_phase(
f"saturation-c{concurrency}", "saturation", alice, chat,
args.base_url, args.duration, concurrency, args.env_file,
saturation_profile,
))
for profile_name in (
"fixed-r1", "fixed-r2", "fixed-r4",
"growth-r1", "growth-r2", "growth-r4",
"improved-db-cpu-r4",
):
profile = all_profiles[profile_name]
print(f"profile: {profile_name}", flush=True)
applied[profile_name] = configure_profile(args.env_file, profile)
warmup_requests += run_warmup(alice, chat, args, profile)
kind = "improvement" if profile_name.startswith("improved") else profile_name.split("-", 1)[0]
phases.append(run_phase(
profile_name, kind, alice, chat, args.base_url,
args.duration, args.comparison_concurrency, args.env_file,
profile,
))
end_dataset = dataset(args.env_file)
finally:
restoration["attempted"] = True
try:
restoration["applied"] = configure_profile(args.env_file, restore)
restoration["succeeded"] = True
except Exception as exc:
restoration["error"] = f"{type(exc).__name__}: {exc}"
if not restoration["succeeded"]:
raise RuntimeError("experiment completed but default Compose profile was not restored")
report = {
"schema_version": 1,
"environment": environment,
"hypothesis": (
"API replicas stop improving throughput when shared PostgreSQL CPU saturates; "
"adding PostgreSQL CPU should improve the unchanged four-replica profile."
),
"methodology": {
"workload": "closed-loop POST messages through the public HAProxy gateway; unique idempotency keys; no client retries",
"duration_seconds_per_phase": args.duration,
"warmup_seconds_per_profile": args.warmup_seconds,
"comparison_concurrency": args.comparison_concurrency,
"saturation_concurrency": sweep,
"fixed_total_budget": "1.00 API CPU, 512 MiB API RAM, 16 total DB pool connections",
"growth_budget": "0.50 API CPU, 256 MiB RAM and 8 DB pool connections per replica",
"improvement": "only PostgreSQL CPU changes from 1.00 to 2.00",
"saturation_rule": "RPS gain below 10% and p95 growth at least 1.5x versus the previous step",
"cpu_bottleneck_rule": "average at least 70% and peak at least 95% of PostgreSQL CPU limit; service-specific threshold for the 0.25 CPU load balancer",
},
"profiles": all_profiles,
"applied_limits": applied,
"phases": phases,
"analysis": analyze(phases),
"dataset": {
"before": start_dataset,
"after": end_dataset,
"warmup_successful_requests": warmup_requests,
},
"restoration": restoration,
"limitations": [
"single Docker host: replicas do not prove host-level availability",
"closed-loop workload has coordinated omission",
"successful-response latency percentiles are reported separately from failures",
"one measured run per profile; host scheduling noise is possible",
],
}
destination = ROOT / args.output
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
try:
displayed_output = destination.relative_to(ROOT)
except ValueError:
displayed_output = destination
print(json.dumps({
"output": str(displayed_output),
"analysis": report["analysis"],
"restoration": restoration["succeeded"],
}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()