forked from mtusi2026/mtusi2026lab3
251 lines
11 KiB
Python
251 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Repeatable 1/2/4-replica resource-scaling experiment."""
|
|
import csv
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import re
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
LOAD = ROOT / "tests" / "load.py"
|
|
OUT = ROOT / "evidence" / "benchmark"
|
|
|
|
|
|
def command(args, env=None, capture=True):
|
|
return subprocess.run(
|
|
args,
|
|
cwd=ROOT,
|
|
env=env,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE if capture else None,
|
|
stderr=subprocess.PIPE if capture else None,
|
|
)
|
|
|
|
|
|
def docker_resources():
|
|
result = command(["docker", "info", "--format", "{{.NCPU}} {{.MemTotal}}"])
|
|
cpu_count, memory_bytes = result.stdout.strip().split()
|
|
return {"docker_daemon_cpus": int(cpu_count), "docker_daemon_memory_bytes": int(memory_bytes)}
|
|
|
|
|
|
def read_stats(samples, errors, stop):
|
|
while not stop.is_set():
|
|
try:
|
|
containers = command(
|
|
["docker", "compose", "--env-file", ".env", "ps", "-q"]
|
|
).stdout.split()
|
|
if not containers:
|
|
stop.wait(1)
|
|
continue
|
|
output = command(
|
|
[
|
|
"docker",
|
|
"stats",
|
|
"--no-stream",
|
|
"--format",
|
|
"{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}",
|
|
*containers,
|
|
]
|
|
).stdout
|
|
for line in output.splitlines():
|
|
fields = line.split("|")
|
|
if len(fields) != 3:
|
|
continue
|
|
name, cpu, memory = fields
|
|
used = re.fullmatch(
|
|
r"([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)",
|
|
memory.split("/")[0].strip(),
|
|
)
|
|
if not used:
|
|
continue
|
|
amount = float(used.group(1))
|
|
unit = used.group(2).lower()
|
|
scale = {"b": 1, "kib": 1024, "mib": 1024**2, "gib": 1024**3}.get(unit)
|
|
if scale is not None:
|
|
samples.append(
|
|
{
|
|
"container": name,
|
|
"cpu_percent": float(cpu.rstrip("%")),
|
|
"memory_bytes": int(amount * scale),
|
|
}
|
|
)
|
|
except subprocess.CalledProcessError as exc:
|
|
errors.append("docker stats failed: " + (exc.stderr or "").strip())
|
|
except (OSError, ValueError) as exc:
|
|
errors.append("docker stats sample failed: " + str(exc))
|
|
stop.wait(1)
|
|
|
|
|
|
def graph(rows, destination):
|
|
width, height = 900, 450
|
|
left, top, chart_width, chart_height = 80, 55, 770, 310
|
|
max_value = max((row["mean_successful_rps"] for row in rows), default=1) or 1
|
|
bar_width = chart_width / max(1, len(rows)) * 0.56
|
|
elements = [
|
|
'<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d" viewBox="0 0 %d %d">'
|
|
% (width, height, width, height),
|
|
'<rect width="100%" height="100%" fill="white"/>',
|
|
'<text x="450" y="28" text-anchor="middle" font-family="sans-serif" font-size="18">Successful message writes per second</text>',
|
|
'<line x1="%d" y1="%d" x2="%d" y2="%d" stroke="#333"/>'
|
|
% (left, top + chart_height, left + chart_width, top + chart_height),
|
|
]
|
|
for index, row in enumerate(rows):
|
|
x = left + (index + 0.5) * chart_width / len(rows) - bar_width / 2
|
|
bar_height = chart_height * row["mean_successful_rps"] / max_value
|
|
y = top + chart_height - bar_height
|
|
elements.append(
|
|
'<rect x="%.1f" y="%.1f" width="%.1f" height="%.1f" fill="#2878b5"/>'
|
|
% (x, y, bar_width, bar_height)
|
|
)
|
|
elements.append(
|
|
'<text x="%.1f" y="%.1f" text-anchor="middle" font-family="sans-serif" font-size="12">%.1f</text>'
|
|
% (x + bar_width / 2, y - 6, row["mean_successful_rps"])
|
|
)
|
|
label = "%d rep %s" % (row["replicas"], row["scenario"])
|
|
elements.append(
|
|
'<text x="%.1f" y="%d" text-anchor="middle" font-family="sans-serif" font-size="11">%s</text>'
|
|
% (x + bar_width / 2, top + chart_height + 22, label)
|
|
)
|
|
elements.extend(["</svg>", ""])
|
|
destination.write_text("\n".join(elements), encoding="utf-8")
|
|
|
|
|
|
def main():
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--duration", type=int, default=10)
|
|
parser.add_argument("--concurrency", type=int, default=8)
|
|
parser.add_argument("--repeats", type=int, default=3)
|
|
args = parser.parse_args()
|
|
if not 5 <= args.duration <= 3600 or not 1 <= args.concurrency <= 128 or args.repeats < 3:
|
|
parser.error("use duration >=5s, concurrency 1..128, and at least 3 repeats")
|
|
if not (ROOT / ".env").is_file() or not (ROOT / "secrets" / "database_password").is_file():
|
|
parser.error("create local .env and secrets/database_password as documented in REPORT.md")
|
|
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
configs = [
|
|
("fixed", 1, "1.00", "512M"),
|
|
("fixed", 2, "0.50", "256M"),
|
|
("fixed", 4, "0.25", "128M"),
|
|
("growth", 2, "1.00", "512M"),
|
|
("growth", 4, "0.50", "256M"),
|
|
]
|
|
results = []
|
|
machine = {"platform": platform.platform(), **docker_resources()}
|
|
print(json.dumps({"machine": machine, "duration_seconds": args.duration, "concurrency": args.concurrency, "repeats": args.repeats}, indent=2))
|
|
|
|
for scenario, replicas, cpu, memory in configs:
|
|
env = os.environ.copy()
|
|
env.update({"API_REPLICAS": str(replicas), "API_CPUS": cpu, "API_MEMORY": memory})
|
|
command(
|
|
[
|
|
"docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait",
|
|
"--wait-timeout", "120", "--scale", "api=" + str(replicas),
|
|
],
|
|
env=env,
|
|
)
|
|
per_run = []
|
|
for repeat in range(1, args.repeats + 1):
|
|
slug = "%s-r%d-%s-%d" % (scenario, replicas, cpu.replace(".", "p"), repeat)
|
|
metrics_path = OUT / (slug + ".json")
|
|
samples = []
|
|
sample_errors = []
|
|
stop = threading.Event()
|
|
sampler = threading.Thread(
|
|
target=read_stats, args=(samples, sample_errors, stop), daemon=True
|
|
)
|
|
sampler.start()
|
|
try:
|
|
command(
|
|
[
|
|
sys.executable, str(LOAD), "--base-url", "http://localhost:8080",
|
|
"--duration", str(args.duration), "--concurrency", str(args.concurrency),
|
|
"--output", str(metrics_path),
|
|
]
|
|
)
|
|
finally:
|
|
stop.set()
|
|
sampler.join(timeout=3)
|
|
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
|
|
metrics.update(
|
|
{
|
|
"scenario": scenario,
|
|
"replicas": replicas,
|
|
"api_cpu_limit_each": cpu,
|
|
"api_memory_limit_each": memory,
|
|
"run": repeat,
|
|
"resource_samples": len(samples),
|
|
"resource_sampling_errors": sample_errors,
|
|
"peak_sampled_api_cpu_percent": max(
|
|
(sample["cpu_percent"] for sample in samples if "-api-" in sample["container"]),
|
|
default=None,
|
|
),
|
|
"peak_sampled_api_memory_bytes": max(
|
|
(sample["memory_bytes"] for sample in samples if "-api-" in sample["container"]),
|
|
default=None,
|
|
),
|
|
"peak_sampled_postgres_cpu_percent": max(
|
|
(sample["cpu_percent"] for sample in samples if "-postgres-" in sample["container"]),
|
|
default=None,
|
|
),
|
|
"peak_sampled_postgres_memory_bytes": max(
|
|
(sample["memory_bytes"] for sample in samples if "-postgres-" in sample["container"]),
|
|
default=None,
|
|
),
|
|
}
|
|
)
|
|
metrics_path.write_text(json.dumps(metrics, indent=2) + "\n", encoding="utf-8")
|
|
per_run.append(metrics)
|
|
print(json.dumps({key: metrics[key] for key in ("scenario", "replicas", "run", "successful_rps", "error_ratio", "successful_latency_ms")}))
|
|
|
|
results.append(
|
|
{
|
|
"scenario": scenario,
|
|
"replicas": replicas,
|
|
"api_cpu_limit_each": cpu,
|
|
"api_memory_limit_each": memory,
|
|
"mean_successful_rps": round(statistics.mean(item["successful_rps"] for item in per_run), 3),
|
|
"stdev_successful_rps": round(statistics.stdev(item["successful_rps"] for item in per_run), 3),
|
|
"mean_error_ratio": round(statistics.mean(item["error_ratio"] for item in per_run), 6),
|
|
"mean_p50_ms": round(statistics.mean(item["successful_latency_ms"]["p50"] for item in per_run), 3),
|
|
"mean_p95_ms": round(statistics.mean(item["successful_latency_ms"]["p95"] for item in per_run), 3),
|
|
"mean_p99_ms": round(statistics.mean(item["successful_latency_ms"]["p99"] for item in per_run), 3),
|
|
"max_sampled_api_cpu_percent": max(
|
|
(item["peak_sampled_api_cpu_percent"] for item in per_run if item["peak_sampled_api_cpu_percent"] is not None),
|
|
default=None,
|
|
),
|
|
"max_sampled_api_memory_bytes": max(
|
|
(item["peak_sampled_api_memory_bytes"] for item in per_run if item["peak_sampled_api_memory_bytes"] is not None),
|
|
default=None,
|
|
),
|
|
"max_sampled_postgres_cpu_percent": max(
|
|
(item["peak_sampled_postgres_cpu_percent"] for item in per_run if item["peak_sampled_postgres_cpu_percent"] is not None),
|
|
default=None,
|
|
),
|
|
"max_sampled_postgres_memory_bytes": max(
|
|
(item["peak_sampled_postgres_memory_bytes"] for item in per_run if item["peak_sampled_postgres_memory_bytes"] is not None),
|
|
default=None,
|
|
),
|
|
}
|
|
)
|
|
|
|
(OUT / "machine.json").write_text(json.dumps(machine, indent=2) + "\n", encoding="utf-8")
|
|
(OUT / "summary.json").write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8")
|
|
with (OUT / "summary.csv").open("w", newline="", encoding="utf-8") as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=list(results[0]))
|
|
writer.writeheader()
|
|
writer.writerows(results)
|
|
graph(results, OUT / "throughput.svg")
|
|
print("Saved raw trial data and summary in " + str(OUT))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|