import argparse import json import os import subprocess import sys import time import urllib.error import urllib.request from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] RAW = ROOT / "results" / "scaling" LOAD_DIR = RAW / "load" STATS_DIR = RAW / "stats" BASE_URL = os.getenv("BENCH_BASE_URL", "http://localhost:8080") CONCURRENCY = [8, 16, 32, 64] DURATION = 30 WARMUP = 10 REPEATS = 2 NGINX_BASELINE = { "KEEPALIVE_DIRECTIVE": "", "PROXY_HTTP_VERSION_DIRECTIVE": "", "PROXY_CONNECTION_DIRECTIVE": "", } NGINX_KEEPALIVE = { "KEEPALIVE_DIRECTIVE": "keepalive 64;", "PROXY_HTTP_VERSION_DIRECTIVE": "proxy_http_version 1.1;", "PROXY_CONNECTION_DIRECTIVE": 'proxy_set_header Connection "";', } def series_a(): return [ dict(name="A_r1", replicas=1, api_cpu=2.0, api_mem="1024M", pg_cpu=1.0, pool=10, overflow=20, pool_timeout=30, nginx=NGINX_BASELINE), dict(name="A_r2", replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0, pool=10, overflow=20, pool_timeout=30, nginx=NGINX_BASELINE), dict(name="A_r4", replicas=4, api_cpu=0.5, api_mem="256M", pg_cpu=1.0, pool=10, overflow=20, pool_timeout=30, nginx=NGINX_BASELINE), ] def series_b(): return [ dict(name="B_cpu1", replicas=2, api_cpu=0.5, api_mem="512M", pg_cpu=1.0, pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE), dict(name="B_cpu2", replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0, pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE), dict(name="B_cpu4", replicas=2, api_cpu=2.0, api_mem="512M", pg_cpu=1.0, pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE), dict(name="B_cpu4_db2", replicas=2, api_cpu=2.0, api_mem="512M", pg_cpu=2.0, pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE), ] def series_c(): base = dict(replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0, pool=10, overflow=20, pool_timeout=30) return [ dict(name="C_before", **base, nginx=NGINX_BASELINE), dict(name="C_after", **base, nginx=NGINX_KEEPALIVE), ] def series_d(): base = dict(replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0, nginx=NGINX_KEEPALIVE) return [ dict(name="D_pool_starved", **base, pool=2, overflow=0, pool_timeout=5), dict(name="D_pool_normal", **base, pool=10, overflow=20, pool_timeout=30), ] SERIES = {"A": series_a, "B": series_b, "C": series_c, "D": series_d} def build_env(cfg): env = os.environ.copy() env["PYTHONUTF8"] = "1" env.update({ "API_REPLICAS": str(cfg["replicas"]), "API_CPU": str(cfg["api_cpu"]), "API_MEM": cfg["api_mem"], "PG_CPU": str(cfg["pg_cpu"]), "DB_POOL_SIZE": str(cfg["pool"]), "DB_MAX_OVERFLOW": str(cfg["overflow"]), "DB_POOL_TIMEOUT": str(cfg.get("pool_timeout", 30)), "DB_POOL_RECYCLE": str(cfg.get("pool_recycle", 3600)), }) env.update(cfg["nginx"]) return env def run(cmd, env, check=True, quiet=False): return subprocess.run( cmd, cwd=ROOT, env=env, check=check, stdout=subprocess.DEVNULL if quiet else None, stderr=subprocess.STDOUT if quiet else None, ) def wait_ready(timeout=180): deadline = time.time() + timeout while time.time() < deadline: try: with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as r: if r.status == 200: return True except (urllib.error.URLError, urllib.error.HTTPError, OSError, TimeoutError): pass time.sleep(2) return False def observe_instances(n=14): seen = set() for _ in range(n): try: with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as r: v = r.headers.get("X-Instance-Id") if v: seen.add(v) except Exception: pass return sorted(seen) def psql(sql, env): p = subprocess.run( ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", env.get("POSTGRES_USER", "mtusi"), "-d", env.get("POSTGRES_DB", "mtusi-messenger"), "-At", "-c", sql], cwd=ROOT, env=env, capture_output=True, text=True, ) return p.stdout.strip() if p.returncode == 0 else "" def pg_snapshot(env): out = { "numbackends": psql("SELECT count(*) FROM pg_stat_activity WHERE datname IS NOT NULL;", env), "active": psql("SELECT count(*) FROM pg_stat_activity WHERE state='active';", env), "waits": psql("SELECT coalesce(wait_event_type,'cpu')||':'||count(*) FROM pg_stat_activity " "WHERE backend_type='client backend' GROUP BY 1 ORDER BY 2 DESC;", env), "top_sql": psql("SELECT calls||'|'||round(total_exec_time::numeric,1)||'|'||left(query,70) " "FROM pg_stat_statements WHERE query ILIKE '%messages%' " "AND query NOT ILIKE '%pg_stat%' ORDER BY total_exec_time DESC LIMIT 5;", env), } return out class StatsSampler: def __init__(self, path, interval=2.0): self.path = path self.interval = interval self.proc = None self.fh = None def __enter__(self): self.path.parent.mkdir(parents=True, exist_ok=True) self.fh = open(self.path, "w", encoding="utf-8") fmt = '"{{.Name}}","{{.CPUPerc}}","{{.MemUsage}}","{{.NetIO}}","{{.BlockIO}}"' script = (f'while true; do docker stats --no-stream --format {fmt}; ' f'echo "---SAMPLE---"; sleep {self.interval}; done') self.proc = subprocess.Popen(["bash", "-c", script], cwd=ROOT, stdout=self.fh, stderr=subprocess.DEVNULL) return self def __exit__(self, *a): if self.proc: self.proc.terminate() try: self.proc.wait(timeout=5) except subprocess.TimeoutExpired: self.proc.kill() if self.fh: self.fh.close() def load_once(cfg, concurrency, duration, tag, env): out = LOAD_DIR / f"{cfg['name']}_c{concurrency}_{tag}.json" out.parent.mkdir(parents=True, exist_ok=True) run([sys.executable, "-X", "utf8", "tests/load.py", "--base-url", BASE_URL, "--duration", str(duration), "--concurrency", str(concurrency), "--output", str(out)], env, check=False) if not out.exists(): return None return json.loads(out.read_text(encoding="utf-8")) def apply_config(cfg, env): print(f"\n\nCONFIG {cfg['name']}: replicas={cfg['replicas']} " f"cpu/replica={cfg['api_cpu']} Σcpu={cfg['replicas']*cfg['api_cpu']:.1f} " f"pg_cpu={cfg['pg_cpu']} " f"pool={cfg['pool']}+{cfg['overflow']} " f"(max {cfg['replicas']*(cfg['pool']+cfg['overflow'])} conns) " f"timeout={cfg.get('pool_timeout',30)}s\n") run(["docker", "compose", "up", "-d", "--no-deps", "--force-recreate", "--scale", f"api={cfg['replicas']}", "api"], env, quiet=True) run(["docker", "compose", "up", "-d", "--no-deps", "--force-recreate", "nginx"], env, quiet=True) if not wait_ready(): raise RuntimeError(f"{cfg['name']}: /health/ready not reached") time.sleep(5) ids = observe_instances() print(f"Observed X-Instance-Id ({len(ids)}): {ids}") return ids def main(): ap = argparse.ArgumentParser() ap.add_argument("--series", nargs="+", default=["A"], choices=list(SERIES)) ap.add_argument("--concurrency", nargs="+", type=int, default=CONCURRENCY) ap.add_argument("--duration", type=int, default=DURATION) ap.add_argument("--repeats", type=int, default=REPEATS) args = ap.parse_args() RAW.mkdir(parents=True, exist_ok=True) results = { "started_at": datetime.now(timezone.utc).isoformat(), "base_url": BASE_URL, "profile": {"duration_s": args.duration, "warmup_s": WARMUP, "concurrency": args.concurrency, "repeats": args.repeats, "tool": "tests/load.py (closed-loop, no retries)"}, "configs": [], } for s in args.series: for cfg in SERIES[s](): env = build_env(cfg) ids = apply_config(cfg, env) entry = {"series": s, **{k: v for k, v in cfg.items() if k != "nginx"}, "nginx_mode": "keepalive" if cfg["nginx"]["KEEPALIVE_DIRECTIVE"] else "baseline", "observed_instance_ids": ids, "points": []} print(f"Warmup {WARMUP}s @ c={args.concurrency[0]} ...") load_once(cfg, args.concurrency[0], WARMUP, "warmup", env) for c in args.concurrency: runs = [] for r in range(args.repeats): pg_before = pg_snapshot(env) with StatsSampler(STATS_DIR / f"{cfg['name']}_c{c}_r{r}.csv"): m = load_once(cfg, c, args.duration, f"r{r}", env) pg_after = pg_snapshot(env) if m is None: print(f" c={c} r={r}: LOAD FAILED") continue runs.append(m) print(f" c={c} r={r}: rps={m['successful_rps']} " f"err={m['error_ratio']} p95={m['successful_latency_ms']['p95']}ms " f"pg_backends={pg_after['numbackends']}") entry["points"].append({ "concurrency": c, "repeat": r, "metrics": m, "pg_before": pg_before, "pg_after": pg_after, "stats_csv": str(STATS_DIR / f"{cfg['name']}_c{c}_r{r}.csv"), }) if runs: rps = sorted(x["successful_rps"] for x in runs) entry.setdefault("summary", {})[str(c)] = { "median_rps": rps[len(rps)//2], "runs": len(runs), } results["configs"].append(entry) (RAW / "raw.json").write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8") results["finished_at"] = datetime.now(timezone.utc).isoformat() (RAW / "raw.json").write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8") print(f"\nDONE. Raw data: {RAW/'raw.json'}") if __name__ == "__main__": main()