Files
mtusi2026lab3/scripts/benchmark.py
2026-11-15 22:00:21 +03:00

251 lines
11 KiB
Python

#!/usr/bin/env python3
"""Repeatable 1/2/4-replica resource-scaling experiment."""
import csv
import json
import os
from pathlib import Path
import platform
import re
import statistics
import subprocess
import sys
import threading
ROOT = Path(__file__).resolve().parents[1]
LOAD = ROOT / "tests" / "load.py"
OUT = ROOT / "evidence" / "benchmark"
def command(args, env=None, capture=True):
return subprocess.run(
args,
cwd=ROOT,
env=env,
check=True,
text=True,
stdout=subprocess.PIPE if capture else None,
stderr=subprocess.PIPE if capture else None,
)
def docker_resources():
result = command(["docker", "info", "--format", "{{.NCPU}} {{.MemTotal}}"])
cpu_count, memory_bytes = result.stdout.strip().split()
return {"docker_daemon_cpus": int(cpu_count), "docker_daemon_memory_bytes": int(memory_bytes)}
def read_stats(samples, errors, stop):
while not stop.is_set():
try:
containers = command(
["docker", "compose", "--env-file", ".env", "ps", "-q"]
).stdout.split()
if not containers:
stop.wait(1)
continue
output = command(
[
"docker",
"stats",
"--no-stream",
"--format",
"{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}",
*containers,
]
).stdout
for line in output.splitlines():
fields = line.split("|")
if len(fields) != 3:
continue
name, cpu, memory = fields
used = re.fullmatch(
r"([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)",
memory.split("/")[0].strip(),
)
if not used:
continue
amount = float(used.group(1))
unit = used.group(2).lower()
scale = {"b": 1, "kib": 1024, "mib": 1024**2, "gib": 1024**3}.get(unit)
if scale is not None:
samples.append(
{
"container": name,
"cpu_percent": float(cpu.rstrip("%")),
"memory_bytes": int(amount * scale),
}
)
except subprocess.CalledProcessError as exc:
errors.append("docker stats failed: " + (exc.stderr or "").strip())
except (OSError, ValueError) as exc:
errors.append("docker stats sample failed: " + str(exc))
stop.wait(1)
def graph(rows, destination):
width, height = 900, 450
left, top, chart_width, chart_height = 80, 55, 770, 310
max_value = max((row["mean_successful_rps"] for row in rows), default=1) or 1
bar_width = chart_width / max(1, len(rows)) * 0.56
elements = [
'<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d" viewBox="0 0 %d %d">'
% (width, height, width, height),
'<rect width="100%" height="100%" fill="white"/>',
'<text x="450" y="28" text-anchor="middle" font-family="sans-serif" font-size="18">Successful message writes per second</text>',
'<line x1="%d" y1="%d" x2="%d" y2="%d" stroke="#333"/>'
% (left, top + chart_height, left + chart_width, top + chart_height),
]
for index, row in enumerate(rows):
x = left + (index + 0.5) * chart_width / len(rows) - bar_width / 2
bar_height = chart_height * row["mean_successful_rps"] / max_value
y = top + chart_height - bar_height
elements.append(
'<rect x="%.1f" y="%.1f" width="%.1f" height="%.1f" fill="#2878b5"/>'
% (x, y, bar_width, bar_height)
)
elements.append(
'<text x="%.1f" y="%.1f" text-anchor="middle" font-family="sans-serif" font-size="12">%.1f</text>'
% (x + bar_width / 2, y - 6, row["mean_successful_rps"])
)
label = "%d rep %s" % (row["replicas"], row["scenario"])
elements.append(
'<text x="%.1f" y="%d" text-anchor="middle" font-family="sans-serif" font-size="11">%s</text>'
% (x + bar_width / 2, top + chart_height + 22, label)
)
elements.extend(["</svg>", ""])
destination.write_text("\n".join(elements), encoding="utf-8")
def main():
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--duration", type=int, default=10)
parser.add_argument("--concurrency", type=int, default=8)
parser.add_argument("--repeats", type=int, default=3)
args = parser.parse_args()
if not 5 <= args.duration <= 3600 or not 1 <= args.concurrency <= 128 or args.repeats < 3:
parser.error("use duration >=5s, concurrency 1..128, and at least 3 repeats")
if not (ROOT / ".env").is_file() or not (ROOT / "secrets" / "database_password").is_file():
parser.error("create local .env and secrets/database_password as documented in REPORT.md")
OUT.mkdir(parents=True, exist_ok=True)
configs = [
("fixed", 1, "1.00", "512M"),
("fixed", 2, "0.50", "256M"),
("fixed", 4, "0.25", "128M"),
("growth", 2, "1.00", "512M"),
("growth", 4, "0.50", "256M"),
]
results = []
machine = {"platform": platform.platform(), **docker_resources()}
print(json.dumps({"machine": machine, "duration_seconds": args.duration, "concurrency": args.concurrency, "repeats": args.repeats}, indent=2))
for scenario, replicas, cpu, memory in configs:
env = os.environ.copy()
env.update({"API_REPLICAS": str(replicas), "API_CPUS": cpu, "API_MEMORY": memory})
command(
[
"docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait",
"--wait-timeout", "120", "--scale", "api=" + str(replicas),
],
env=env,
)
per_run = []
for repeat in range(1, args.repeats + 1):
slug = "%s-r%d-%s-%d" % (scenario, replicas, cpu.replace(".", "p"), repeat)
metrics_path = OUT / (slug + ".json")
samples = []
sample_errors = []
stop = threading.Event()
sampler = threading.Thread(
target=read_stats, args=(samples, sample_errors, stop), daemon=True
)
sampler.start()
try:
command(
[
sys.executable, str(LOAD), "--base-url", "http://localhost:8080",
"--duration", str(args.duration), "--concurrency", str(args.concurrency),
"--output", str(metrics_path),
]
)
finally:
stop.set()
sampler.join(timeout=3)
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics.update(
{
"scenario": scenario,
"replicas": replicas,
"api_cpu_limit_each": cpu,
"api_memory_limit_each": memory,
"run": repeat,
"resource_samples": len(samples),
"resource_sampling_errors": sample_errors,
"peak_sampled_api_cpu_percent": max(
(sample["cpu_percent"] for sample in samples if "-api-" in sample["container"]),
default=None,
),
"peak_sampled_api_memory_bytes": max(
(sample["memory_bytes"] for sample in samples if "-api-" in sample["container"]),
default=None,
),
"peak_sampled_postgres_cpu_percent": max(
(sample["cpu_percent"] for sample in samples if "-postgres-" in sample["container"]),
default=None,
),
"peak_sampled_postgres_memory_bytes": max(
(sample["memory_bytes"] for sample in samples if "-postgres-" in sample["container"]),
default=None,
),
}
)
metrics_path.write_text(json.dumps(metrics, indent=2) + "\n", encoding="utf-8")
per_run.append(metrics)
print(json.dumps({key: metrics[key] for key in ("scenario", "replicas", "run", "successful_rps", "error_ratio", "successful_latency_ms")}))
results.append(
{
"scenario": scenario,
"replicas": replicas,
"api_cpu_limit_each": cpu,
"api_memory_limit_each": memory,
"mean_successful_rps": round(statistics.mean(item["successful_rps"] for item in per_run), 3),
"stdev_successful_rps": round(statistics.stdev(item["successful_rps"] for item in per_run), 3),
"mean_error_ratio": round(statistics.mean(item["error_ratio"] for item in per_run), 6),
"mean_p50_ms": round(statistics.mean(item["successful_latency_ms"]["p50"] for item in per_run), 3),
"mean_p95_ms": round(statistics.mean(item["successful_latency_ms"]["p95"] for item in per_run), 3),
"mean_p99_ms": round(statistics.mean(item["successful_latency_ms"]["p99"] for item in per_run), 3),
"max_sampled_api_cpu_percent": max(
(item["peak_sampled_api_cpu_percent"] for item in per_run if item["peak_sampled_api_cpu_percent"] is not None),
default=None,
),
"max_sampled_api_memory_bytes": max(
(item["peak_sampled_api_memory_bytes"] for item in per_run if item["peak_sampled_api_memory_bytes"] is not None),
default=None,
),
"max_sampled_postgres_cpu_percent": max(
(item["peak_sampled_postgres_cpu_percent"] for item in per_run if item["peak_sampled_postgres_cpu_percent"] is not None),
default=None,
),
"max_sampled_postgres_memory_bytes": max(
(item["peak_sampled_postgres_memory_bytes"] for item in per_run if item["peak_sampled_postgres_memory_bytes"] is not None),
default=None,
),
}
)
(OUT / "machine.json").write_text(json.dumps(machine, indent=2) + "\n", encoding="utf-8")
(OUT / "summary.json").write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8")
with (OUT / "summary.csv").open("w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=list(results[0]))
writer.writeheader()
writer.writerows(results)
graph(results, OUT / "throughput.svg")
print("Saved raw trial data and summary in " + str(OUT))
if __name__ == "__main__":
main()