Files
mtusi2026lab3/scripts/scaling_benchmark.py
T

278 lines
10 KiB
Python

import argparse
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
RAW = ROOT / "results" / "scaling"
LOAD_DIR = RAW / "load"
STATS_DIR = RAW / "stats"
BASE_URL = os.getenv("BENCH_BASE_URL", "http://localhost:8080")
CONCURRENCY = [8, 16, 32, 64]
DURATION = 30
WARMUP = 10
REPEATS = 2
NGINX_BASELINE = {
"KEEPALIVE_DIRECTIVE": "",
"PROXY_HTTP_VERSION_DIRECTIVE": "",
"PROXY_CONNECTION_DIRECTIVE": "",
}
NGINX_KEEPALIVE = {
"KEEPALIVE_DIRECTIVE": "keepalive 64;",
"PROXY_HTTP_VERSION_DIRECTIVE": "proxy_http_version 1.1;",
"PROXY_CONNECTION_DIRECTIVE": 'proxy_set_header Connection "";',
}
def series_a():
return [
dict(name="A_r1", replicas=1, api_cpu=2.0, api_mem="1024M", pg_cpu=1.0,
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_BASELINE),
dict(name="A_r2", replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0,
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_BASELINE),
dict(name="A_r4", replicas=4, api_cpu=0.5, api_mem="256M", pg_cpu=1.0,
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_BASELINE),
]
def series_b():
return [
dict(name="B_cpu1", replicas=2, api_cpu=0.5, api_mem="512M", pg_cpu=1.0,
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE),
dict(name="B_cpu2", replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0,
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE),
dict(name="B_cpu4", replicas=2, api_cpu=2.0, api_mem="512M", pg_cpu=1.0,
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE),
dict(name="B_cpu4_db2", replicas=2, api_cpu=2.0, api_mem="512M", pg_cpu=2.0,
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE),
]
def series_c():
base = dict(replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0,
pool=10, overflow=20, pool_timeout=30)
return [
dict(name="C_before", **base, nginx=NGINX_BASELINE),
dict(name="C_after", **base, nginx=NGINX_KEEPALIVE),
]
def series_d():
base = dict(replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0,
nginx=NGINX_KEEPALIVE)
return [
dict(name="D_pool_starved", **base, pool=2, overflow=0, pool_timeout=5),
dict(name="D_pool_normal", **base, pool=10, overflow=20, pool_timeout=30),
]
SERIES = {"A": series_a, "B": series_b, "C": series_c, "D": series_d}
def build_env(cfg):
env = os.environ.copy()
env["PYTHONUTF8"] = "1"
env.update({
"API_REPLICAS": str(cfg["replicas"]),
"API_CPU": str(cfg["api_cpu"]),
"API_MEM": cfg["api_mem"],
"PG_CPU": str(cfg["pg_cpu"]),
"DB_POOL_SIZE": str(cfg["pool"]),
"DB_MAX_OVERFLOW": str(cfg["overflow"]),
"DB_POOL_TIMEOUT": str(cfg.get("pool_timeout", 30)),
"DB_POOL_RECYCLE": str(cfg.get("pool_recycle", 3600)),
})
env.update(cfg["nginx"])
return env
def run(cmd, env, check=True, quiet=False):
return subprocess.run(
cmd, cwd=ROOT, env=env, check=check,
stdout=subprocess.DEVNULL if quiet else None,
stderr=subprocess.STDOUT if quiet else None,
)
def wait_ready(timeout=180):
deadline = time.time() + timeout
while time.time() < deadline:
try:
with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as r:
if r.status == 200:
return True
except (urllib.error.URLError, urllib.error.HTTPError, OSError, TimeoutError):
pass
time.sleep(2)
return False
def observe_instances(n=14):
seen = set()
for _ in range(n):
try:
with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as r:
v = r.headers.get("X-Instance-Id")
if v:
seen.add(v)
except Exception:
pass
return sorted(seen)
def psql(sql, env):
p = subprocess.run(
["docker", "compose", "exec", "-T", "postgres", "psql",
"-U", env.get("POSTGRES_USER", "mtusi"),
"-d", env.get("POSTGRES_DB", "mtusi-messenger"), "-At", "-c", sql],
cwd=ROOT, env=env, capture_output=True, text=True,
)
return p.stdout.strip() if p.returncode == 0 else ""
def pg_snapshot(env):
out = {
"numbackends": psql("SELECT count(*) FROM pg_stat_activity WHERE datname IS NOT NULL;", env),
"active": psql("SELECT count(*) FROM pg_stat_activity WHERE state='active';", env),
"waits": psql("SELECT coalesce(wait_event_type,'cpu')||':'||count(*) FROM pg_stat_activity "
"WHERE backend_type='client backend' GROUP BY 1 ORDER BY 2 DESC;", env),
"top_sql": psql("SELECT calls||'|'||round(total_exec_time::numeric,1)||'|'||left(query,70) "
"FROM pg_stat_statements WHERE query ILIKE '%messages%' "
"AND query NOT ILIKE '%pg_stat%' ORDER BY total_exec_time DESC LIMIT 5;", env),
}
return out
class StatsSampler:
def __init__(self, path, interval=2.0):
self.path = path
self.interval = interval
self.proc = None
self.fh = None
def __enter__(self):
self.path.parent.mkdir(parents=True, exist_ok=True)
self.fh = open(self.path, "w", encoding="utf-8")
fmt = '"{{.Name}}","{{.CPUPerc}}","{{.MemUsage}}","{{.NetIO}}","{{.BlockIO}}"'
script = (f'while true; do docker stats --no-stream --format {fmt}; '
f'echo "---SAMPLE---"; sleep {self.interval}; done')
self.proc = subprocess.Popen(["bash", "-c", script], cwd=ROOT,
stdout=self.fh, stderr=subprocess.DEVNULL)
return self
def __exit__(self, *a):
if self.proc:
self.proc.terminate()
try:
self.proc.wait(timeout=5)
except subprocess.TimeoutExpired:
self.proc.kill()
if self.fh:
self.fh.close()
def load_once(cfg, concurrency, duration, tag, env):
out = LOAD_DIR / f"{cfg['name']}_c{concurrency}_{tag}.json"
out.parent.mkdir(parents=True, exist_ok=True)
run([sys.executable, "-X", "utf8", "tests/load.py",
"--base-url", BASE_URL,
"--duration", str(duration),
"--concurrency", str(concurrency),
"--output", str(out)], env, check=False)
if not out.exists():
return None
return json.loads(out.read_text(encoding="utf-8"))
def apply_config(cfg, env):
print(f"\n\nCONFIG {cfg['name']}: replicas={cfg['replicas']} "
f"cpu/replica={cfg['api_cpu']} Σcpu={cfg['replicas']*cfg['api_cpu']:.1f} "
f"pg_cpu={cfg['pg_cpu']} "
f"pool={cfg['pool']}+{cfg['overflow']} "
f"(max {cfg['replicas']*(cfg['pool']+cfg['overflow'])} conns) "
f"timeout={cfg.get('pool_timeout',30)}s\n")
run(["docker", "compose", "up", "-d", "--no-deps", "--force-recreate",
"--scale", f"api={cfg['replicas']}", "api"], env, quiet=True)
run(["docker", "compose", "up", "-d", "--no-deps", "--force-recreate", "nginx"], env, quiet=True)
if not wait_ready():
raise RuntimeError(f"{cfg['name']}: /health/ready not reached")
time.sleep(5)
ids = observe_instances()
print(f"Observed X-Instance-Id ({len(ids)}): {ids}")
return ids
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--series", nargs="+", default=["A"], choices=list(SERIES))
ap.add_argument("--concurrency", nargs="+", type=int, default=CONCURRENCY)
ap.add_argument("--duration", type=int, default=DURATION)
ap.add_argument("--repeats", type=int, default=REPEATS)
args = ap.parse_args()
RAW.mkdir(parents=True, exist_ok=True)
results = {
"started_at": datetime.now(timezone.utc).isoformat(),
"base_url": BASE_URL,
"profile": {"duration_s": args.duration, "warmup_s": WARMUP,
"concurrency": args.concurrency, "repeats": args.repeats,
"tool": "tests/load.py (closed-loop, no retries)"},
"configs": [],
}
for s in args.series:
for cfg in SERIES[s]():
env = build_env(cfg)
ids = apply_config(cfg, env)
entry = {"series": s, **{k: v for k, v in cfg.items() if k != "nginx"},
"nginx_mode": "keepalive" if cfg["nginx"]["KEEPALIVE_DIRECTIVE"] else "baseline",
"observed_instance_ids": ids, "points": []}
print(f"Warmup {WARMUP}s @ c={args.concurrency[0]} ...")
load_once(cfg, args.concurrency[0], WARMUP, "warmup", env)
for c in args.concurrency:
runs = []
for r in range(args.repeats):
pg_before = pg_snapshot(env)
with StatsSampler(STATS_DIR / f"{cfg['name']}_c{c}_r{r}.csv"):
m = load_once(cfg, c, args.duration, f"r{r}", env)
pg_after = pg_snapshot(env)
if m is None:
print(f" c={c} r={r}: LOAD FAILED")
continue
runs.append(m)
print(f" c={c} r={r}: rps={m['successful_rps']} "
f"err={m['error_ratio']} p95={m['successful_latency_ms']['p95']}ms "
f"pg_backends={pg_after['numbackends']}")
entry["points"].append({
"concurrency": c, "repeat": r, "metrics": m,
"pg_before": pg_before, "pg_after": pg_after,
"stats_csv": str(STATS_DIR / f"{cfg['name']}_c{c}_r{r}.csv"),
})
if runs:
rps = sorted(x["successful_rps"] for x in runs)
entry.setdefault("summary", {})[str(c)] = {
"median_rps": rps[len(rps)//2],
"runs": len(runs),
}
results["configs"].append(entry)
(RAW / "raw.json").write_text(json.dumps(results, ensure_ascii=False, indent=2),
encoding="utf-8")
results["finished_at"] = datetime.now(timezone.utc).isoformat()
(RAW / "raw.json").write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
print(f"\nDONE. Raw data: {RAW/'raw.json'}")
if __name__ == "__main__":
main()