This commit is contained in:
2026-10-25 21:46:21 +03:00
parent 5300a1a1e0
commit a7d6487e2d
33 changed files with 5965 additions and 556 deletions
+3 -3
View File
@@ -77,7 +77,7 @@ def memory_mib(raw):
def service(name):
for candidate in ("load-balancer", "postgres", "redis", "api"):
for candidate in ("load-balancer", "frontend", "postgres", "redis", "api"):
if f"-{candidate}-" in name:
return candidate
return "other"
@@ -357,7 +357,7 @@ def main():
"action_after_seconds": args.action_after_seconds,
"post_action_seconds": args.post_action_seconds,
"warmup_requests": 12,
"workload": "closed-loop POST messages via Nginx; no automatic retries",
"workload": "closed-loop POST messages via the public HAProxy gateway; no automatic retries",
},
"phases": phases,
"idempotency": idem,
@@ -371,7 +371,7 @@ def main():
"single Docker host",
"closed-loop coordinated omission",
"successful-response latency percentiles only",
"PostgreSQL and Nginx remain single points of failure",
"PostgreSQL and HAProxy remain single points of failure",
],
}
destination = ROOT / args.output
+577
View File
@@ -0,0 +1,577 @@
#!/usr/bin/env python3
"""Measure the price of horizontal API scaling for lab 3 grade 5."""
import argparse
from collections import Counter, defaultdict
import datetime as dt
import json
import math
import os
from pathlib import Path
import platform
import subprocess
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
def command(args, env=None, timeout=240, check=True):
result = subprocess.run(
[str(value) for value in args],
cwd=ROOT,
env=env,
text=True,
capture_output=True,
timeout=timeout,
check=False,
)
if check and result.returncode:
detail = (result.stderr or result.stdout).strip().splitlines()
suffix = f": {detail[-1]}" if detail else ""
raise RuntimeError(
f"{' '.join(map(str, args[:4]))} exited {result.returncode}{suffix}"
)
return result
def compose(env_file, *args, env=None, timeout=240, check=True):
return command(
["docker", "compose", "--env-file", env_file, *args],
env=env,
timeout=timeout,
check=check,
)
def inspect(container_id):
raw = command(
["docker", "inspect", "--format", "{{json .}}", container_id]
).stdout
return json.loads(raw)
def service_ids(env_file, service=None):
args = ["ps", "-q"]
if service:
args.append(service)
return [line for line in compose(env_file, *args).stdout.splitlines() if line]
def wait_healthy(env_file, service, expected=1, timeout=120):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
healthy = []
for container_id in service_ids(env_file, service):
state = inspect(container_id)["State"]
if state.get("Health", {}).get("Status") == "healthy":
healthy.append(container_id)
if len(healthy) == expected:
return healthy
time.sleep(0.5)
raise RuntimeError(f"{service}: expected {expected} healthy containers")
def percentile(values, ratio):
if not values:
return None
ordered = sorted(values)
return round(ordered[max(0, math.ceil(ratio * len(ordered)) - 1)], 3)
def memory_mib(raw):
value = raw.split("/", 1)[0].strip()
units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
for suffix in ("GiB", "MiB", "KiB", "B"):
if value.endswith(suffix):
return round(float(value[:-len(suffix)]) * units[suffix] / 1024**2, 3)
return 0.0
def profile_environment(profile):
env = dict(os.environ)
env.update({
"API_REPLICAS": str(profile["replicas"]),
"API_CPU_LIMIT": f"{profile['api_cpu']:.2f}",
"API_MEMORY_LIMIT": profile["api_memory"],
"DB_MAX_CONNS": str(profile["db_max_conns"]),
"DB_MIN_CONNS": str(profile["db_min_conns"]),
"POSTGRES_CPU_LIMIT": f"{profile['postgres_cpu']:.2f}",
"POSTGRES_MEMORY_LIMIT": profile["postgres_memory"],
})
return env
def configure_profile(env_file, profile):
env = profile_environment(profile)
compose(
env_file,
"up", "-d", "--no-deps", "postgres",
env=env,
timeout=180,
)
wait_healthy(env_file, "postgres")
compose(
env_file,
"up", "-d", "--no-deps",
"--scale", f"api={profile['replicas']}",
"--force-recreate", "api",
env=env,
timeout=240,
)
api = wait_healthy(env_file, "api", profile["replicas"])
wait_healthy(env_file, "load-balancer")
time.sleep(1.5)
expected_cpu = int(profile["api_cpu"] * 1_000_000_000)
expected_memory = parse_memory_limit(profile["api_memory"])
applied = []
for container_id in api:
data = inspect(container_id)
actual_env = dict(item.split("=", 1) for item in data["Config"]["Env"] if "=" in item)
host = data["HostConfig"]
if host.get("NanoCpus") != expected_cpu or host.get("Memory") != expected_memory:
raise RuntimeError("API container limits do not match the selected profile")
if actual_env.get("DB_MAX_CONNS") != str(profile["db_max_conns"]):
raise RuntimeError("API DB pool size does not match the selected profile")
applied.append({
"instance": data["Config"]["Hostname"],
"cpu_limit": host.get("NanoCpus", 0) / 1_000_000_000,
"memory_limit_mib": round(host.get("Memory", 0) / 1024**2, 1),
"db_max_conns": int(actual_env["DB_MAX_CONNS"]),
})
postgres = inspect(service_ids(env_file, "postgres")[0])
expected_pg_cpu = int(profile["postgres_cpu"] * 1_000_000_000)
if postgres["HostConfig"].get("NanoCpus") != expected_pg_cpu:
raise RuntimeError("PostgreSQL CPU limit does not match the selected profile")
return {
"api": sorted(applied, key=lambda item: item["instance"]),
"postgres_cpu_limit": postgres["HostConfig"].get("NanoCpus", 0) / 1_000_000_000,
"postgres_memory_limit_mib": round(postgres["HostConfig"].get("Memory", 0) / 1024**2, 1),
}
def parse_memory_limit(value):
suffix = value[-1].upper()
multiplier = {"K": 1024, "M": 1024**2, "G": 1024**3}[suffix]
return int(value[:-1]) * multiplier
def resource_targets(env_file):
targets = {}
for container_id in service_ids(env_file):
data = inspect(container_id)
service = data["Config"]["Labels"].get("com.docker.compose.service", "other")
if service in {"api", "postgres", "redis", "load-balancer", "frontend"}:
targets[container_id] = {
"service": service,
"name": data["Name"].lstrip("/"),
}
return targets
def stats_snapshot(targets, elapsed):
if not targets:
return {"t": round(elapsed, 3), "services": {}}
result = command(
["docker", "stats", "--no-stream", "--format", "{{json .}}", *targets],
timeout=15,
check=False,
)
grouped = defaultdict(lambda: {"cpu_percent": 0.0, "memory_mib": 0.0, "containers": 0})
for line in result.stdout.splitlines():
try:
row = json.loads(line)
container_id = row.get("ID", "")
match = next((key for key in targets if key.startswith(container_id)), None)
if not match:
continue
values = grouped[targets[match]["service"]]
values["cpu_percent"] += float(row.get("CPUPerc", "0%").rstrip("%"))
values["memory_mib"] += memory_mib(row.get("MemUsage", "0B / 0B"))
values["containers"] += 1
except (ValueError, json.JSONDecodeError):
continue
return {
"t": round(elapsed, 3),
"services": {
name: {
"cpu_percent": round(values["cpu_percent"], 3),
"memory_mib": round(values["memory_mib"], 3),
"containers": values["containers"],
}
for name, values in sorted(grouped.items())
},
}
def summarize_resources(snapshots):
grouped = defaultdict(lambda: {"cpu": [], "memory": [], "containers": []})
for snapshot in snapshots:
for service, values in snapshot["services"].items():
grouped[service]["cpu"].append(values["cpu_percent"])
grouped[service]["memory"].append(values["memory_mib"])
grouped[service]["containers"].append(values["containers"])
return {
service: {
"cpu_percent_avg": round(sum(values["cpu"]) / len(values["cpu"]), 3),
"cpu_percent_max": round(max(values["cpu"]), 3),
"memory_mib_avg": round(sum(values["memory"]) / len(values["memory"]), 3),
"memory_mib_max": round(max(values["memory"]), 3),
"containers": max(values["containers"]),
}
for service, values in sorted(grouped.items())
if values["cpu"]
}
def run_phase(name, kind, actor, chat, base_url, duration, concurrency, env_file, profile):
records = []
snapshots = []
lock = threading.Lock()
stop = threading.Event()
barrier = threading.Barrier(concurrency + 1)
targets = resource_targets(env_file)
started = time.monotonic()
def worker(worker_id):
client = Client(base_url, timeout=5)
index = 0
barrier.wait()
while not stop.is_set():
request_started = time.monotonic()
try:
response = client.message(
actor,
chat,
f"grade5-{name}-{worker_id}-{index}",
)
row = {
"t": round(time.monotonic() - started, 3),
"ok": True,
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
"instance": response["headers"].get("X-Instance-Id", ""),
}
except Failure as exc:
row = {
"t": round(time.monotonic() - started, 3),
"ok": False,
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
"error": str(exc),
}
with lock:
records.append(row)
index += 1
def sampler():
while not stop.is_set():
snapshots.append(stats_snapshot(targets, time.monotonic() - started))
stop.wait(1)
workers = [
threading.Thread(target=worker, args=(index,), daemon=True)
for index in range(concurrency)
]
for thread in workers:
thread.start()
stats_thread = threading.Thread(target=sampler, daemon=True)
stats_thread.start()
started = time.monotonic()
barrier.wait()
time.sleep(duration)
stop.set()
for thread in workers:
thread.join(10)
stats_thread.join(15)
elapsed = time.monotonic() - started
successes = [row for row in records if row["ok"]]
failures = [row for row in records if not row["ok"]]
latencies = [row["latency_ms"] for row in successes]
buckets = defaultdict(lambda: {"successful": 0, "failed": 0})
for row in records:
key = str(max(0, int(row["t"])))
buckets[key]["successful" if row["ok"] else "failed"] += 1
total = len(records)
return {
"name": name,
"kind": kind,
"profile": profile["name"],
"replicas": profile["replicas"],
"concurrency": concurrency,
"elapsed_seconds": round(elapsed, 3),
"successful_requests": len(successes),
"failed_requests": len(failures),
"error_ratio": round(len(failures) / total, 6) if total else None,
"errors": dict(sorted(Counter(row["error"] for row in failures).items())),
"successful_rps": round(len(successes) / elapsed, 3),
"successful_latency_ms": {
"p50": percentile(latencies, 0.50),
"p95": percentile(latencies, 0.95),
"p99": percentile(latencies, 0.99),
},
"instances": sorted({row["instance"] for row in successes if row.get("instance")}),
"per_second": dict(sorted(buckets.items(), key=lambda item: int(item[0]))),
"resource_samples": snapshots,
"resources": summarize_resources(snapshots),
}
def dataset(env_file):
sql = (
"select count(*)::text || '|' || pg_database_size(current_database())::text "
"from messages"
)
result = compose(
env_file,
"exec", "-T", "postgres", "sh", "-c",
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"',
"sh", sql,
)
count, size = result.stdout.strip().split("|", 1)
return {"messages": int(count), "database_size_bytes": int(size)}
def host_environment(env_file):
mem_total_kib = 0
with open("/proc/meminfo", encoding="utf-8") as source:
for line in source:
if line.startswith("MemTotal:"):
mem_total_kib = int(line.split()[1])
break
images = compose(env_file, "config", "--images").stdout.splitlines()
return {
"utc_started_at": dt.datetime.now(dt.timezone.utc).isoformat(),
"git_commit": command(["git", "rev-parse", "HEAD"]).stdout.strip(),
"working_tree_dirty": bool(command(["git", "status", "--porcelain"]).stdout.strip()),
"logical_cpu_count": os.cpu_count(),
"host_memory_mib": round(mem_total_kib / 1024, 1),
"kernel": platform.release(),
"python": platform.python_version(),
"docker_engine": command(["docker", "version", "--format", "{{.Server.Version}}"]).stdout.strip(),
"docker_compose": command(["docker", "compose", "version", "--short"]).stdout.strip(),
"images": images,
}
def analyze(phases):
sweep = sorted(
(phase for phase in phases if phase["kind"] == "saturation"),
key=lambda phase: phase["concurrency"],
)
saturation = None
previous = None
for phase in sweep:
if previous:
gain = (phase["successful_rps"] - previous["successful_rps"]) / max(previous["successful_rps"], 0.001)
latency_growth = phase["successful_latency_ms"]["p95"] / max(previous["successful_latency_ms"]["p95"], 0.001)
if saturation is None and gain < 0.10 and latency_growth >= 1.50:
saturation = {
"concurrency": phase["concurrency"],
"previous_concurrency": previous["concurrency"],
"rps_gain_percent": round(gain * 100, 2),
"p95_growth_ratio": round(latency_growth, 3),
}
previous = phase
baseline = next(phase for phase in phases if phase["name"] == "growth-r4")
improved = next(phase for phase in phases if phase["name"] == "improved-db-cpu-r4")
rps_change = (improved["successful_rps"] - baseline["successful_rps"]) / max(baseline["successful_rps"], 0.001) * 100
p95_change = (improved["successful_latency_ms"]["p95"] - baseline["successful_latency_ms"]["p95"]) / max(baseline["successful_latency_ms"]["p95"], 0.001) * 100
confirmed = (
rps_change >= 5
and improved["error_ratio"] <= baseline["error_ratio"] + 0.001
) or p95_change <= -10
resources = baseline["resources"]
pg = resources.get("postgres", {})
lb = resources.get("load-balancer", {})
bottleneck = "undetermined"
if pg.get("cpu_percent_avg", 0) >= 70 and pg.get("cpu_percent_max", 0) >= 95:
bottleneck = "postgres_cpu"
elif lb.get("cpu_percent_avg", 0) >= 20 and lb.get("cpu_percent_max", 0) >= 24:
bottleneck = "load_balancer_cpu"
improved_resources = improved["resources"]
improved_lb = improved_resources.get("load-balancer", {})
remaining_bottleneck = "undetermined"
if improved_lb.get("cpu_percent_avg", 0) >= 20 and improved_lb.get("cpu_percent_max", 0) >= 24:
remaining_bottleneck = "load_balancer_cpu"
return {
"saturation_point": saturation,
"observed_bottleneck": bottleneck,
"baseline_resource_snapshot": resources,
"improved_resource_snapshot": improved_resources,
"remaining_bottleneck_after_improvement": remaining_bottleneck,
"improvement": {
"change": "PostgreSQL CPU limit: 1.00 -> 2.00; API profile unchanged",
"baseline_phase": baseline["name"],
"improved_phase": improved["name"],
"rps_change_percent": round(rps_change, 2),
"p95_change_percent": round(p95_change, 2),
"confirmed": confirmed,
},
}
def profiles():
base = {"postgres_cpu": 1.0, "postgres_memory": "512M"}
return {
"fixed-r1": {**base, "name": "fixed-r1", "replicas": 1, "api_cpu": 1.0, "api_memory": "512M", "db_max_conns": 16, "db_min_conns": 2},
"fixed-r2": {**base, "name": "fixed-r2", "replicas": 2, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
"fixed-r4": {**base, "name": "fixed-r4", "replicas": 4, "api_cpu": 0.25, "api_memory": "128M", "db_max_conns": 4, "db_min_conns": 1},
"growth-r1": {**base, "name": "growth-r1", "replicas": 1, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
"growth-r2": {**base, "name": "growth-r2", "replicas": 2, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
"growth-r4": {**base, "name": "growth-r4", "replicas": 4, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
"improved-db-cpu-r4": {**base, "name": "improved-db-cpu-r4", "replicas": 4, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1, "postgres_cpu": 2.0},
}
def run_warmup(actor, chat, args, profile):
phase = run_phase(
f"warmup-{profile['name']}",
"warmup",
actor,
chat,
args.base_url,
args.warmup_seconds,
min(4, args.comparison_concurrency),
args.env_file,
profile,
)
if phase["failed_requests"]:
raise RuntimeError(f"warmup failed for {profile['name']}")
return phase["successful_requests"]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--env-file", default=".env")
parser.add_argument("--duration", type=float, default=10)
parser.add_argument("--warmup-seconds", type=float, default=2)
parser.add_argument("--comparison-concurrency", type=int, default=32)
parser.add_argument("--sweep", default="1,2,4,8,16,32")
parser.add_argument("--output", default="evidence/grade5/results.json")
args = parser.parse_args()
sweep = [int(value) for value in args.sweep.split(",")]
if args.duration <= 0 or args.warmup_seconds <= 0:
raise SystemExit("durations must be positive")
if not sweep or min(sweep) < 1 or max(sweep + [args.comparison_concurrency]) > 64:
raise SystemExit("concurrency must be 1..64")
all_profiles = profiles()
restore = {
"name": "restore-default",
"replicas": 2,
"api_cpu": 0.5,
"api_memory": "256M",
"db_max_conns": 10,
"db_min_conns": 2,
"postgres_cpu": 1.0,
"postgres_memory": "512M",
}
phases = []
applied = {}
warmup_requests = 0
restoration = {"attempted": False, "succeeded": False}
environment = host_environment(args.env_file)
start_dataset = dataset(args.env_file)
try:
client = Client(args.base_url, timeout=8)
alice, _, _, chat = client.bootstrap()
saturation_profile = all_profiles["growth-r2"]
applied[saturation_profile["name"]] = configure_profile(args.env_file, saturation_profile)
warmup_requests += run_warmup(alice, chat, args, saturation_profile)
for concurrency in sweep:
print(f"saturation: concurrency={concurrency}", flush=True)
phases.append(run_phase(
f"saturation-c{concurrency}", "saturation", alice, chat,
args.base_url, args.duration, concurrency, args.env_file,
saturation_profile,
))
for profile_name in (
"fixed-r1", "fixed-r2", "fixed-r4",
"growth-r1", "growth-r2", "growth-r4",
"improved-db-cpu-r4",
):
profile = all_profiles[profile_name]
print(f"profile: {profile_name}", flush=True)
applied[profile_name] = configure_profile(args.env_file, profile)
warmup_requests += run_warmup(alice, chat, args, profile)
kind = "improvement" if profile_name.startswith("improved") else profile_name.split("-", 1)[0]
phases.append(run_phase(
profile_name, kind, alice, chat, args.base_url,
args.duration, args.comparison_concurrency, args.env_file,
profile,
))
end_dataset = dataset(args.env_file)
finally:
restoration["attempted"] = True
try:
restoration["applied"] = configure_profile(args.env_file, restore)
restoration["succeeded"] = True
except Exception as exc:
restoration["error"] = f"{type(exc).__name__}: {exc}"
if not restoration["succeeded"]:
raise RuntimeError("experiment completed but default Compose profile was not restored")
report = {
"schema_version": 1,
"environment": environment,
"hypothesis": (
"API replicas stop improving throughput when shared PostgreSQL CPU saturates; "
"adding PostgreSQL CPU should improve the unchanged four-replica profile."
),
"methodology": {
"workload": "closed-loop POST messages through the public HAProxy gateway; unique idempotency keys; no client retries",
"duration_seconds_per_phase": args.duration,
"warmup_seconds_per_profile": args.warmup_seconds,
"comparison_concurrency": args.comparison_concurrency,
"saturation_concurrency": sweep,
"fixed_total_budget": "1.00 API CPU, 512 MiB API RAM, 16 total DB pool connections",
"growth_budget": "0.50 API CPU, 256 MiB RAM and 8 DB pool connections per replica",
"improvement": "only PostgreSQL CPU changes from 1.00 to 2.00",
"saturation_rule": "RPS gain below 10% and p95 growth at least 1.5x versus the previous step",
"cpu_bottleneck_rule": "average at least 70% and peak at least 95% of PostgreSQL CPU limit; service-specific threshold for the 0.25 CPU load balancer",
},
"profiles": all_profiles,
"applied_limits": applied,
"phases": phases,
"analysis": analyze(phases),
"dataset": {
"before": start_dataset,
"after": end_dataset,
"warmup_successful_requests": warmup_requests,
},
"restoration": restoration,
"limitations": [
"single Docker host: replicas do not prove host-level availability",
"closed-loop workload has coordinated omission",
"successful-response latency percentiles are reported separately from failures",
"one measured run per profile; host scheduling noise is possible",
],
}
destination = ROOT / args.output
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
try:
displayed_output = destination.relative_to(ROOT)
except ValueError:
displayed_output = destination
print(json.dumps({
"output": str(displayed_output),
"analysis": report["analysis"],
"restoration": restoration["succeeded"],
}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+2 -2
View File
@@ -148,7 +148,7 @@ def summary(report, output):
"## Method", "",
f"- concurrency: {method['concurrency']};",
f"- baseline duration: {method['baseline_seconds']} s;",
"- load: closed-loop POST messages through Nginx;",
"- load: closed-loop POST messages through the public HAProxy gateway;",
"- failover: both original API containers stopped and restored in turn;",
"- rolling update: one surge replica, old replicas replaced one by one;",
"- percentiles cover successful responses; failures are separate.", "",
@@ -178,7 +178,7 @@ def summary(report, output):
"![Resources](resources.svg)", "",
"## Scope", "",
"Single-host process failure and rolling replacement only. PostgreSQL "
"and Nginx remain single points of failure.", "",
"and HAProxy remain single points of failure.", "",
]
output.write_text("\n".join(lines))
+322
View File
@@ -0,0 +1,322 @@
#!/usr/bin/env python3
"""Render grade-5 scaling evidence as SVG, CSV and Markdown."""
import argparse
import csv
import html
import json
from pathlib import Path
STYLE = """
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
.title { font-size: 26px; font-weight: 650; }
.subtitle { font-size: 14px; fill: #aeb2b8; }
.label { font-size: 14px; fill: #c8cbd0; }
.small { font-size: 12px; fill: #a2a5aa; }
.grid { stroke: #34373d; stroke-width: 1; }
.axis { stroke: #747981; stroke-width: 1.5; }
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
.line2 { fill: none; stroke: #92979f; stroke-width: 3; }
.point { fill: #f4f5f6; }
.point2 { fill: #92979f; }
.bar { fill: #d7d9dc; }
.bar2 { fill: #777c84; }
.bar3 { fill: #4c5057; }
.threshold { stroke: #8f949c; stroke-width: 2; stroke-dasharray: 7 5; }
"""
def svg(title, subtitle, body, width=1120, height=620):
return f"""<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" viewBox="0 0 {width} {height}">
<rect width="100%" height="100%" fill="#0e1013"/>
<style>{STYLE}</style>
<text x="50" y="44" class="title">{html.escape(title)}</text>
<text x="50" y="70" class="subtitle">{html.escape(subtitle)}</text>
{body}
</svg>
"""
def grid(maximum, left, top, width, height, suffix=""):
maximum = maximum or 1
parts = []
for step in range(6):
value = maximum * step / 5
y = top + height - height * step / 5
parts.append(f'<line x1="{left}" y1="{y:.1f}" x2="{left+width}" y2="{y:.1f}" class="grid"/>')
parts.append(f'<text x="{left-12}" y="{y+4:.1f}" text-anchor="end" class="small">{value:.0f}{suffix}</text>')
return parts
def phase_map(report):
return {phase["name"]: phase for phase in report["phases"]}
def saturation_graph(report, output):
phases = sorted(
(phase for phase in report["phases"] if phase["kind"] == "saturation"),
key=lambda phase: phase["concurrency"],
)
left, top, width, height = 90, 110, 940, 390
max_rps = max(phase["successful_rps"] for phase in phases) * 1.1
max_p95 = max(phase["successful_latency_ms"]["p95"] for phase in phases) * 1.1
parts = grid(max_rps, left, top, width, height)
points_rps, points_p95 = [], []
for index, phase in enumerate(phases):
x = left + width * index / max(1, len(phases) - 1)
y_rps = top + height - height * phase["successful_rps"] / max_rps
y_p95 = top + height - height * phase["successful_latency_ms"]["p95"] / max_p95
points_rps.append(f"{x:.1f},{y_rps:.1f}")
points_p95.append(f"{x:.1f},{y_p95:.1f}")
parts.append(f'<circle cx="{x:.1f}" cy="{y_rps:.1f}" r="6" class="point"/>')
parts.append(f'<circle cx="{x:.1f}" cy="{y_p95:.1f}" r="6" class="point2"/>')
parts.append(f'<text x="{x:.1f}" y="{top+height+28}" text-anchor="middle" class="label">{phase["concurrency"]}</text>')
parts.append(f'<text x="{x:.1f}" y="{y_rps-12:.1f}" text-anchor="middle" class="small">{phase["successful_rps"]:.0f}</text>')
parts.append(f'<polyline points="{" ".join(points_rps)}" class="line"/>')
parts.append(f'<polyline points="{" ".join(points_p95)}" class="line2"/>')
parts.append(f'<text x="{left}" y="{top+height+62}" class="label">Concurrency</text>')
parts.append(f'<text x="{left+720}" y="{top+height+62}" class="label">white: RPS · gray: p95 normalized</text>')
parts.append(f'<text x="30" y="{top+height/2}" transform="rotate(-90 30 {top+height/2})" class="label">Successful requests per second</text>')
output.write_text(svg(
"Saturation curve",
"Two API replicas; 1 CPU total; 16 total DB connections; PostgreSQL limited to 1 CPU",
"\n".join(parts),
), encoding="utf-8")
def comparison_graph(report, kind, output):
phases = sorted(
(phase for phase in report["phases"] if phase["kind"] == kind),
key=lambda phase: phase["replicas"],
)
left, top, width, height = 90, 110, 940, 370
max_rps = max(phase["successful_rps"] for phase in phases) * 1.15
parts = grid(max_rps, left, top, width, height)
slot = width / len(phases)
for index, phase in enumerate(phases):
bar_width = min(150, slot * 0.52)
x = left + index * slot + (slot - bar_width) / 2
bar_height = height * phase["successful_rps"] / max_rps
y = top + height - bar_height
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="14" class="bar"/>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-12:.1f}" text-anchor="middle" class="label">{phase["successful_rps"]:.1f} RPS</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+30}" text-anchor="middle" class="label">{phase["replicas"]} replica(s)</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+52}" text-anchor="middle" class="small">p95 {phase["successful_latency_ms"]["p95"]:.1f} ms</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+70}" text-anchor="middle" class="small">errors {phase["error_ratio"]*100:.3f}%</text>')
if kind == "fixed":
title = "Fixed total API budget"
subtitle = "1 CPU · 512 MiB · 16 DB connections split across 1, 2 and 4 replicas"
else:
title = "Growing API budget"
subtitle = "Each replica adds 0.5 CPU · 256 MiB · 8 DB connections; PostgreSQL remains at 1 CPU"
output.write_text(svg(title, subtitle, "\n".join(parts)), encoding="utf-8")
def bottleneck_graph(report, output):
phases = sorted(
(phase for phase in report["phases"] if phase["kind"] == "growth"),
key=lambda phase: phase["replicas"],
)
services = ("api", "postgres", "load-balancer", "frontend", "redis")
left, top, width, height = 90, 110, 940, 370
maximum = max(
phase["resources"].get(service, {}).get("cpu_percent_avg", 0)
for phase in phases for service in services
) * 1.15 or 1
parts = grid(maximum, left, top, width, height)
slot = width / len(phases)
colors = ("bar", "bar2", "bar3", "bar3", "bar3")
for phase_index, phase in enumerate(phases):
group_left = left + phase_index * slot
bar_width = min(48, slot / 6)
for service_index, service in enumerate(services):
value = phase["resources"].get(service, {}).get("cpu_percent_avg", 0)
x = group_left + slot * 0.12 + service_index * (bar_width + 9)
bar_height = height * value / maximum
y = top + height - bar_height
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="7" class="{colors[service_index]}"/>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-7:.1f}" text-anchor="middle" class="small">{value:.0f}</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+22}" text-anchor="middle" class="small">{service[:3]}</text>')
parts.append(f'<text x="{group_left+slot/2:.1f}" y="{top+height+52}" text-anchor="middle" class="label">{phase["replicas"]} replica(s)</text>')
parts.append(f'<text x="30" y="{top+height/2}" transform="rotate(-90 30 {top+height/2})" class="label">Average aggregate CPU, %</text>')
output.write_text(svg(
"Where scaling stops",
"Aggregate service CPU during the proportional-growth series (api · postgres · HAProxy · frontend · redis)",
"\n".join(parts),
), encoding="utf-8")
def improvement_graph(report, output):
phases = phase_map(report)
baseline = phases["growth-r4"]
improved = phases["improved-db-cpu-r4"]
pair = [baseline, improved]
left, top, panel_width, height = 90, 120, 390, 340
parts = []
panels = [
("Successful RPS", "successful_rps", left),
("p95 latency, ms", "p95", 630),
]
for title, metric, panel_left in panels:
values = [
phase["successful_rps"] if metric == "successful_rps" else phase["successful_latency_ms"]["p95"]
for phase in pair
]
maximum = max(values) * 1.2 or 1
parts.append(f'<text x="{panel_left}" y="{top-22}" class="label">{title}</text>')
for index, (phase, value) in enumerate(zip(pair, values)):
bar_width = 120
x = panel_left + 35 + index * 185
bar_height = height * value / maximum
y = top + height - bar_height
parts.append(f'<rect x="{x}" y="{y:.1f}" width="{bar_width}" height="{bar_height:.1f}" rx="14" class="{"bar" if index else "bar2"}"/>')
parts.append(f'<text x="{x+60}" y="{y-10:.1f}" text-anchor="middle" class="label">{value:.1f}</text>')
parts.append(f'<text x="{x+60}" y="{top+height+28}" text-anchor="middle" class="small">PG {1 if index == 0 else 2} CPU</text>')
analysis = report["analysis"]["improvement"]
parts.append(f'<text x="90" y="545" class="label">RPS change: {analysis["rps_change_percent"]:+.2f}% · p95 change: {analysis["p95_change_percent"]:+.2f}% · confirmed: {str(analysis["confirmed"]).lower()}</text>')
output.write_text(svg(
"Confirmed improvement",
"Four API replicas unchanged; only the PostgreSQL CPU limit changes from 1 to 2",
"\n".join(parts),
), encoding="utf-8")
def write_csv(report, destination):
with (destination / "measurements.csv").open("w", newline="", encoding="utf-8") as stream:
fields = [
"phase", "kind", "replicas", "concurrency", "elapsed_seconds",
"successful_requests", "failed_requests", "error_ratio", "successful_rps",
"p50_ms", "p95_ms", "p99_ms", "api_cpu_avg", "postgres_cpu_avg",
"load_balancer_cpu_avg", "frontend_cpu_avg", "redis_cpu_avg",
]
writer = csv.DictWriter(stream, fieldnames=fields)
writer.writeheader()
for phase in report["phases"]:
writer.writerow({
"phase": phase["name"],
"kind": phase["kind"],
"replicas": phase["replicas"],
"concurrency": phase["concurrency"],
"elapsed_seconds": phase["elapsed_seconds"],
"successful_requests": phase["successful_requests"],
"failed_requests": phase["failed_requests"],
"error_ratio": phase["error_ratio"],
"successful_rps": phase["successful_rps"],
"p50_ms": phase["successful_latency_ms"]["p50"],
"p95_ms": phase["successful_latency_ms"]["p95"],
"p99_ms": phase["successful_latency_ms"]["p99"],
"api_cpu_avg": phase["resources"].get("api", {}).get("cpu_percent_avg"),
"postgres_cpu_avg": phase["resources"].get("postgres", {}).get("cpu_percent_avg"),
"load_balancer_cpu_avg": phase["resources"].get("load-balancer", {}).get("cpu_percent_avg"),
"frontend_cpu_avg": phase["resources"].get("frontend", {}).get("cpu_percent_avg"),
"redis_cpu_avg": phase["resources"].get("redis", {}).get("cpu_percent_avg"),
})
with (destination / "timeseries.csv").open("w", newline="", encoding="utf-8") as stream:
fields = ["phase", "second", "successful", "failed"]
writer = csv.DictWriter(stream, fieldnames=fields)
writer.writeheader()
for phase in report["phases"]:
for second, counts in phase["per_second"].items():
writer.writerow({"phase": phase["name"], "second": second, **counts})
with (destination / "resources.csv").open("w", newline="", encoding="utf-8") as stream:
fields = ["phase", "t", "service", "containers", "cpu_percent", "memory_mib"]
writer = csv.DictWriter(stream, fieldnames=fields)
writer.writeheader()
for phase in report["phases"]:
for snapshot in phase["resource_samples"]:
for service, values in snapshot["services"].items():
writer.writerow({
"phase": phase["name"], "t": snapshot["t"], "service": service,
**values,
})
def markdown_summary(report, output):
method = report["methodology"]
analysis = report["analysis"]
improvement = analysis["improvement"]
lines = [
"# Lab 3 grade-5 scaling experiment", "",
f"Generated: {report['environment']['utc_started_at']}.", "",
"## Hypothesis", "", report["hypothesis"], "",
"## Reproduction", "",
f"- workload: {method['workload']};",
f"- phase duration: {method['duration_seconds_per_phase']} s;",
f"- comparison concurrency: {method['comparison_concurrency']};",
f"- saturation sweep: {method['saturation_concurrency']};",
f"- fixed budget: {method['fixed_total_budget']};",
f"- growth series: {method['growth_budget']};",
f"- saturation rule: {method['saturation_rule']};",
f"- host: {report['environment']['logical_cpu_count']} logical CPU, {report['environment']['host_memory_mib']} MiB RAM;",
f"- Docker {report['environment']['docker_engine']}, Compose {report['environment']['docker_compose']};",
"- command: `make grade5-experiment`; warmup runs before every changed profile.", "",
"## Measurements", "",
"| Phase | Replicas | Concurrency | RPS | Errors | p50 ms | p95 ms | p99 ms | PG CPU avg | API CPU avg |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
]
for phase in report["phases"]:
latency = phase["successful_latency_ms"]
resources = phase["resources"]
lines.append(
f"| {phase['name']} | {phase['replicas']} | {phase['concurrency']} | "
f"{phase['successful_rps']} | {phase['failed_requests']} ({phase['error_ratio']*100:.3f}%) | "
f"{latency['p50']} | {latency['p95']} | {latency['p99']} | "
f"{resources.get('postgres', {}).get('cpu_percent_avg', 'n/a')} | "
f"{resources.get('api', {}).get('cpu_percent_avg', 'n/a')} |"
)
saturation = analysis["saturation_point"]
lines += ["", "## Conclusion", ""]
if saturation:
lines.append(
f"The first saturation signal appears at concurrency {saturation['concurrency']}: "
f"RPS changes by {saturation['rps_gain_percent']:+.2f}% while p95 changes by "
f"{saturation['p95_growth_ratio']:.2f}x versus concurrency {saturation['previous_concurrency']}."
)
else:
lines.append("The configured sweep did not locate a saturation point by the declared rule.")
lines += [
"",
f"Observed bottleneck classification: `{analysis['observed_bottleneck']}`.", "",
f"The PostgreSQL CPU change produced {improvement['rps_change_percent']:+.2f}% RPS and "
f"{improvement['p95_change_percent']:+.2f}% p95; confirmed improvement: "
f"`{str(improvement['confirmed']).lower()}`.", "",
f"After the improvement the remaining bottleneck classification is "
f"`{analysis['remaining_bottleneck_after_improvement']}`.", "",
"## Graphs", "",
"![Saturation](01-saturation.svg)", "",
"![Fixed budget](02-fixed-total.svg)", "",
"![Growing budget](03-resource-growth.svg)", "",
"![Bottleneck](04-bottleneck.svg)", "",
"![Improvement](05-improvement.svg)", "",
"## Raw data and limits", "",
"Raw aggregates are in `measurements.csv`; one-second request buckets are in "
"`timeseries.csv`; every resource sample is in `resources.csv`; complete metadata and "
"applied limits are in `results.json`.", "",
]
lines.extend(f"- {item}" for item in report["limitations"])
lines.append("")
output.write_text("\n".join(lines), encoding="utf-8")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("results", nargs="?", default="evidence/grade5/results.json")
args = parser.parse_args()
source = Path(args.results)
report = json.loads(source.read_text(encoding="utf-8"))
destination = source.parent
saturation_graph(report, destination / "01-saturation.svg")
comparison_graph(report, "fixed", destination / "02-fixed-total.svg")
comparison_graph(report, "growth", destination / "03-resource-growth.svg")
bottleneck_graph(report, destination / "04-bottleneck.svg")
improvement_graph(report, destination / "05-improvement.svg")
write_csv(report, destination)
markdown_summary(report, destination / "README.md")
print(f"rendered grade-5 evidence in {destination}")
if __name__ == "__main__":
main()
+5 -1
View File
@@ -19,6 +19,10 @@ compose up -d --no-deps api
compose restart api
compose up -d --wait --no-deps api
# Публичная точка входа перезапускается последней, когда API уже готовы.
# Статический frontend возвращается до публичной точки входа.
compose restart frontend
compose up -d --wait --no-deps frontend
# Публичная точка входа перезапускается последней, когда API и frontend готовы.
compose restart load-balancer
compose up -d --wait --no-deps load-balancer
+198
View File
@@ -0,0 +1,198 @@
#!/usr/bin/env python3
"""Prove that the gateway excludes an API replica which loses readiness."""
import argparse
from collections import Counter
import http.client
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
FAULT_IMAGE = "node:24-alpine@sha256:44fce696f38ba8c987c4db89954051000bbd05ffd8d8659ba15ed4a8602e937f"
def command(args, timeout=180, check=True):
result = subprocess.run(
[str(value) for value in args],
cwd=ROOT,
text=True,
capture_output=True,
timeout=timeout,
check=False,
)
if check and result.returncode:
detail = (result.stderr or result.stdout).strip().splitlines()
suffix = f": {detail[-1]}" if detail else ""
raise RuntimeError(f"{' '.join(map(str, args[:4]))} failed{suffix}")
return result
def compose(env_file, *args):
return command(["docker", "compose", "--env-file", env_file, *args])
def inspect(container_id):
return json.loads(command([
"docker", "inspect", "--format", "{{json .}}", container_id,
]).stdout)
def call(host, port, method, path, body=None, headers=None, timeout=5):
connection = http.client.HTTPConnection(host, port, timeout=timeout)
connection.request(method, path, body=body, headers=headers or {})
response = connection.getresponse()
payload = response.read()
result = {
"status": response.status,
"instance": response.getheader("X-Instance-Id", ""),
"body": payload,
}
connection.close()
return result
def observe_instances(base_host, base_port, requests=16):
instances = Counter()
statuses = Counter()
for _ in range(requests):
result = call(base_host, base_port, "GET", "/health/live")
statuses[result["status"]] += 1
instances[result["instance"]] += 1
return statuses, instances
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--env-file", default=".env")
parser.add_argument("--host", default="localhost")
parser.add_argument("--port", type=int, default=8080)
parser.add_argument("--output", default="")
args = parser.parse_args()
api_ids = [line for line in compose(args.env_file, "ps", "-q", "api").stdout.splitlines() if line]
if len(api_ids) != 2:
raise RuntimeError("readiness routing test requires exactly two API replicas")
target_id = api_ids[0]
target = inspect(target_id)
target_instance = target["Config"]["Hostname"]
target_ip = next(iter(target["NetworkSettings"]["Networks"].values()))["IPAddress"]
postgres_id = compose(args.env_file, "ps", "-q", "postgres").stdout.strip()
postgres = inspect(postgres_id)
postgres_ip = next(iter(postgres["NetworkSettings"]["Networks"].values()))["IPAddress"]
before_statuses, before_instances = observe_instances(args.host, args.port, 40)
if before_statuses != {200: 40} or len(before_instances) != 2:
raise RuntimeError("both API replicas must serve traffic before fault injection")
helper = f"mtusi2026lab3-readiness-fault-{os.getpid()}"
rule = [
"iptables", "-D", "OUTPUT", "-d", postgres_ip,
"-p", "tcp", "--dport", "5432", "-j", "REJECT",
"--reject-with", "tcp-reset",
]
detection_started = time.monotonic()
try:
script = (
"apk add --no-cache iptables >/dev/null && "
f"iptables -I OUTPUT -d {postgres_ip} -p tcp --dport 5432 "
"-j REJECT --reject-with tcp-reset && "
"touch /tmp/fault-ready && sleep 180"
)
command([
"docker", "run", "-d", "--name", helper,
"--network", f"container:{target_id}",
"--cap-add", "NET_ADMIN", FAULT_IMAGE,
"sh", "-c", script,
])
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
ready = command(
["docker", "exec", helper, "test", "-f", "/tmp/fault-ready"],
check=False,
)
if ready.returncode == 0:
break
time.sleep(0.2)
else:
raise RuntimeError("fault helper did not become ready")
direct = call(target_ip, 8080, "GET", "/health/ready")
if direct["status"] != 503:
raise RuntimeError(f"target readiness is {direct['status']}, want 503")
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
statuses, instances = observe_instances(args.host, args.port, 16)
if statuses == {200: 16} and target_instance not in instances:
break
time.sleep(0.25)
else:
raise RuntimeError("gateway did not exclude the unready API within 10 seconds")
detection_seconds = time.monotonic() - detection_started
health_statuses, health_instances = observe_instances(args.host, args.port, 40)
registration_statuses = Counter()
registration_instances = Counter()
for _ in range(20):
body = json.dumps({
"username": "readiness_" + uuid.uuid4().hex[:16],
"display_name": "Readiness audit",
"password": "T3st-" + uuid.uuid4().hex,
}).encode()
response = call(
args.host,
args.port,
"POST",
"/api/v1/users",
body=body,
headers={"Content-Type": "application/json"},
timeout=8,
)
registration_statuses[response["status"]] += 1
registration_instances[response["instance"]] += 1
if health_statuses != {200: 40} or target_instance in health_instances:
raise RuntimeError("unready API received liveness traffic through gateway")
if registration_statuses != {201: 20} or target_instance in registration_instances:
raise RuntimeError("unready API received registration traffic through gateway")
finally:
command(["docker", "exec", helper, *rule], check=False)
command(["docker", "rm", "-f", helper], check=False)
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
direct = call(target_ip, 8080, "GET", "/health/ready")
if direct["status"] == 200:
_, instances = observe_instances(args.host, args.port, 24)
if target_instance in instances:
break
time.sleep(0.25)
else:
raise RuntimeError("recovered API did not return to gateway rotation")
result = {
"target_instance": target_instance,
"detection_seconds": round(detection_seconds, 3),
"health_requests": dict(health_statuses),
"registration_requests": dict(registration_statuses),
"recovered": True,
}
rendered = json.dumps(result, ensure_ascii=False, indent=2)
if args.output:
output = Path(args.output)
if not output.is_absolute():
output = ROOT / output
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(rendered + "\n", encoding="utf-8")
print(rendered)
print("PASS: readiness-aware gateway excluded and restored one API replica")
if __name__ == "__main__":
main()