#!/usr/bin/env python3 """Repeatable 1/2/4-replica resource-scaling experiment.""" import csv import json import os from pathlib import Path import platform import re import statistics import subprocess import sys import threading ROOT = Path(__file__).resolve().parents[1] LOAD = ROOT / "tests" / "load.py" OUT = ROOT / "evidence" / "benchmark" def command(args, env=None, capture=True): return subprocess.run( args, cwd=ROOT, env=env, check=True, text=True, stdout=subprocess.PIPE if capture else None, stderr=subprocess.PIPE if capture else None, ) def docker_resources(): result = command(["docker", "info", "--format", "{{.NCPU}} {{.MemTotal}}"]) cpu_count, memory_bytes = result.stdout.strip().split() return {"docker_daemon_cpus": int(cpu_count), "docker_daemon_memory_bytes": int(memory_bytes)} def read_stats(samples, errors, stop): while not stop.is_set(): try: containers = command( ["docker", "compose", "--env-file", ".env", "ps", "-q"] ).stdout.split() if not containers: stop.wait(1) continue output = command( [ "docker", "stats", "--no-stream", "--format", "{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}", *containers, ] ).stdout for line in output.splitlines(): fields = line.split("|") if len(fields) != 3: continue name, cpu, memory = fields used = re.fullmatch( r"([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)", memory.split("/")[0].strip(), ) if not used: continue amount = float(used.group(1)) unit = used.group(2).lower() scale = {"b": 1, "kib": 1024, "mib": 1024**2, "gib": 1024**3}.get(unit) if scale is not None: samples.append( { "container": name, "cpu_percent": float(cpu.rstrip("%")), "memory_bytes": int(amount * scale), } ) except subprocess.CalledProcessError as exc: errors.append("docker stats failed: " + (exc.stderr or "").strip()) except (OSError, ValueError) as exc: errors.append("docker stats sample failed: " + str(exc)) stop.wait(1) def graph(rows, destination): width, height = 900, 450 left, top, chart_width, chart_height = 80, 55, 770, 310 max_value = max((row["mean_successful_rps"] for row in rows), default=1) or 1 bar_width = chart_width / max(1, len(rows)) * 0.56 elements = [ '' % (width, height, width, height), '', 'Successful message writes per second', '' % (left, top + chart_height, left + chart_width, top + chart_height), ] for index, row in enumerate(rows): x = left + (index + 0.5) * chart_width / len(rows) - bar_width / 2 bar_height = chart_height * row["mean_successful_rps"] / max_value y = top + chart_height - bar_height elements.append( '' % (x, y, bar_width, bar_height) ) elements.append( '%.1f' % (x + bar_width / 2, y - 6, row["mean_successful_rps"]) ) label = "%d rep %s" % (row["replicas"], row["scenario"]) elements.append( '%s' % (x + bar_width / 2, top + chart_height + 22, label) ) elements.extend(["", ""]) destination.write_text("\n".join(elements), encoding="utf-8") def main(): import argparse parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--duration", type=int, default=10) parser.add_argument("--concurrency", type=int, default=8) parser.add_argument("--repeats", type=int, default=3) args = parser.parse_args() if not 5 <= args.duration <= 3600 or not 1 <= args.concurrency <= 128 or args.repeats < 3: parser.error("use duration >=5s, concurrency 1..128, and at least 3 repeats") if not (ROOT / ".env").is_file() or not (ROOT / "secrets" / "database_password").is_file(): parser.error("create local .env and secrets/database_password as documented in REPORT.md") OUT.mkdir(parents=True, exist_ok=True) configs = [ ("fixed", 1, "1.00", "512M"), ("fixed", 2, "0.50", "256M"), ("fixed", 4, "0.25", "128M"), ("growth", 2, "1.00", "512M"), ("growth", 4, "0.50", "256M"), ] results = [] machine = {"platform": platform.platform(), **docker_resources()} print(json.dumps({"machine": machine, "duration_seconds": args.duration, "concurrency": args.concurrency, "repeats": args.repeats}, indent=2)) for scenario, replicas, cpu, memory in configs: env = os.environ.copy() env.update({"API_REPLICAS": str(replicas), "API_CPUS": cpu, "API_MEMORY": memory}) command( [ "docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait", "--wait-timeout", "120", "--scale", "api=" + str(replicas), ], env=env, ) per_run = [] for repeat in range(1, args.repeats + 1): slug = "%s-r%d-%s-%d" % (scenario, replicas, cpu.replace(".", "p"), repeat) metrics_path = OUT / (slug + ".json") samples = [] sample_errors = [] stop = threading.Event() sampler = threading.Thread( target=read_stats, args=(samples, sample_errors, stop), daemon=True ) sampler.start() try: command( [ sys.executable, str(LOAD), "--base-url", "http://localhost:8080", "--duration", str(args.duration), "--concurrency", str(args.concurrency), "--output", str(metrics_path), ] ) finally: stop.set() sampler.join(timeout=3) metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics.update( { "scenario": scenario, "replicas": replicas, "api_cpu_limit_each": cpu, "api_memory_limit_each": memory, "run": repeat, "resource_samples": len(samples), "resource_sampling_errors": sample_errors, "peak_sampled_api_cpu_percent": max( (sample["cpu_percent"] for sample in samples if "-api-" in sample["container"]), default=None, ), "peak_sampled_api_memory_bytes": max( (sample["memory_bytes"] for sample in samples if "-api-" in sample["container"]), default=None, ), "peak_sampled_postgres_cpu_percent": max( (sample["cpu_percent"] for sample in samples if "-postgres-" in sample["container"]), default=None, ), "peak_sampled_postgres_memory_bytes": max( (sample["memory_bytes"] for sample in samples if "-postgres-" in sample["container"]), default=None, ), } ) metrics_path.write_text(json.dumps(metrics, indent=2) + "\n", encoding="utf-8") per_run.append(metrics) print(json.dumps({key: metrics[key] for key in ("scenario", "replicas", "run", "successful_rps", "error_ratio", "successful_latency_ms")})) results.append( { "scenario": scenario, "replicas": replicas, "api_cpu_limit_each": cpu, "api_memory_limit_each": memory, "mean_successful_rps": round(statistics.mean(item["successful_rps"] for item in per_run), 3), "stdev_successful_rps": round(statistics.stdev(item["successful_rps"] for item in per_run), 3), "mean_error_ratio": round(statistics.mean(item["error_ratio"] for item in per_run), 6), "mean_p50_ms": round(statistics.mean(item["successful_latency_ms"]["p50"] for item in per_run), 3), "mean_p95_ms": round(statistics.mean(item["successful_latency_ms"]["p95"] for item in per_run), 3), "mean_p99_ms": round(statistics.mean(item["successful_latency_ms"]["p99"] for item in per_run), 3), "max_sampled_api_cpu_percent": max( (item["peak_sampled_api_cpu_percent"] for item in per_run if item["peak_sampled_api_cpu_percent"] is not None), default=None, ), "max_sampled_api_memory_bytes": max( (item["peak_sampled_api_memory_bytes"] for item in per_run if item["peak_sampled_api_memory_bytes"] is not None), default=None, ), "max_sampled_postgres_cpu_percent": max( (item["peak_sampled_postgres_cpu_percent"] for item in per_run if item["peak_sampled_postgres_cpu_percent"] is not None), default=None, ), "max_sampled_postgres_memory_bytes": max( (item["peak_sampled_postgres_memory_bytes"] for item in per_run if item["peak_sampled_postgres_memory_bytes"] is not None), default=None, ), } ) (OUT / "machine.json").write_text(json.dumps(machine, indent=2) + "\n", encoding="utf-8") (OUT / "summary.json").write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") with (OUT / "summary.csv").open("w", newline="", encoding="utf-8") as handle: writer = csv.DictWriter(handle, fieldnames=list(results[0])) writer.writeheader() writer.writerows(results) graph(results, OUT / "throughput.svg") print("Saved raw trial data and summary in " + str(OUT)) if __name__ == "__main__": main()