#!/usr/bin/env python3 import argparse import json import os import subprocess import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) seed = __import__("lab4_seed") FORMAT = "{:>5} {:>5} {:>4} {:<18} {:>7} {:>7} {:>8} {:>8} {:>7} {:>8} {:>6}" COLUMNS = FORMAT.format( "slots", "size", "conc", "codes", "p50_ms", "p95_ms", "api_idle", "api_peak", "api_oom", "lb_peak", "lb_oom", ) OVERRIDES = ("UPLOAD_CONCURRENCY", "MALLOC_MMAP_THRESHOLD_") def recreate(slots): env = dict(os.environ, UPLOAD_CONCURRENCY=str(slots)) if not slots: env = {k: v for k, v in os.environ.items() if k not in OVERRIDES} cmd = ["docker", "compose", "up", "-d", "--no-deps", "--force-recreate", "--wait", "api", "lb"] subprocess.run(cmd, cwd=ROOT, env=env, check=True, capture_output=True) subprocess.run([str(ROOT / "scripts/lb-wait.sh")], cwd=ROOT, check=True) def balancer(): cid = seed.run(["docker", "compose", "ps", "-q", "lb"]).strip() events = dict(row.split() for row in seed.cgroup(cid, "memory.events").splitlines()) peak = seed.cgroup(cid, "memory.peak") return { "cgroup_memory_peak_mib": round(int(peak) / 1048576, 1), "oom_kill": int(events["oom_kill"]), } def line(report): memory = report["memory"].values() codes = ",".join(f"{k}x{v}" for k, v in sorted(report["status_codes"].items())) return FORMAT.format( report["upload_concurrency"], f"{report['body_bytes'] / 1048576:.1f}M", report["concurrency"], codes, report["latency_ms"]["p50"], report["latency_ms"]["p95"], max(m["idle_mib"] for m in memory), max(m["cgroup_memory_peak_mib"] or 0 for m in memory), sum(m["oom_kill"] for m in memory), report["lb"]["cgroup_memory_peak_mib"], report["lb"]["oom_kill"], ) def main(): p = argparse.ArgumentParser(description="Peak memory of API replicas under concurrent uploads") p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) p.add_argument("--slots", type=int, nargs="+", default=[64, 3]) p.add_argument("--sizes", type=int, nargs="+", default=[1 << 20, 10 << 20]) p.add_argument("--levels", type=int, nargs="+", default=[10, 20, 40]) p.add_argument("--rounds", type=int, default=2, help="uploads per concurrent client") p.add_argument("--out", default="evidence/upload-memory") args = p.parse_args() out = ROOT / args.out out.mkdir(parents=True, exist_ok=True) if any(out.iterdir()): p.error("output directory must be empty; choose a new --out") commit = seed.run(["git", "rev-parse", "HEAD"]).strip() threshold = os.environ.get("MALLOC_MMAP_THRESHOLD_", "131072") lines = [ f"# {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit {commit}; api x2: 1 CPU," f" 512 MiB, 2 workers; MALLOC_MMAP_THRESHOLD_={threshold}; {args.rounds} uploads" " per client; peaks from cgroup memory.peak, MiB", COLUMNS, ] print("\n".join(lines), flush=True) try: for slots in args.slots: for size in args.sizes: for level in args.levels: recreate(slots) run = argparse.Namespace( base_url=args.base_url, timeout=60, size=size, count=level * args.rounds, concurrency=level, sample=True, service="api", link=0, ids_out=None, ) report = { "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": commit, "upload_concurrency": slots, **seed.over_http(run), } report["lb"] = balancer() name = f"slots{slots}-{size >> 20}mib-c{level}.json" (out / name).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") lines.append(line(report)) print(lines[-1], flush=True) finally: recreate(None) (out / "summary.txt").write_text("\n".join(lines) + "\n") if __name__ == "__main__": main()