forked from mtusi2026/mtusi2026lab3
278 lines
10 KiB
Python
278 lines
10 KiB
Python
import argparse
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
RAW = ROOT / "results" / "scaling"
|
|
LOAD_DIR = RAW / "load"
|
|
STATS_DIR = RAW / "stats"
|
|
BASE_URL = os.getenv("BENCH_BASE_URL", "http://localhost:8080")
|
|
|
|
CONCURRENCY = [8, 16, 32, 64]
|
|
DURATION = 30
|
|
WARMUP = 10
|
|
REPEATS = 2
|
|
|
|
NGINX_BASELINE = {
|
|
"KEEPALIVE_DIRECTIVE": "",
|
|
"PROXY_HTTP_VERSION_DIRECTIVE": "",
|
|
"PROXY_CONNECTION_DIRECTIVE": "",
|
|
}
|
|
NGINX_KEEPALIVE = {
|
|
"KEEPALIVE_DIRECTIVE": "keepalive 64;",
|
|
"PROXY_HTTP_VERSION_DIRECTIVE": "proxy_http_version 1.1;",
|
|
"PROXY_CONNECTION_DIRECTIVE": 'proxy_set_header Connection "";',
|
|
}
|
|
|
|
|
|
def series_a():
|
|
return [
|
|
dict(name="A_r1", replicas=1, api_cpu=2.0, api_mem="1024M", pg_cpu=1.0,
|
|
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_BASELINE),
|
|
dict(name="A_r2", replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0,
|
|
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_BASELINE),
|
|
dict(name="A_r4", replicas=4, api_cpu=0.5, api_mem="256M", pg_cpu=1.0,
|
|
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_BASELINE),
|
|
]
|
|
|
|
|
|
def series_b():
|
|
return [
|
|
dict(name="B_cpu1", replicas=2, api_cpu=0.5, api_mem="512M", pg_cpu=1.0,
|
|
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE),
|
|
dict(name="B_cpu2", replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0,
|
|
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE),
|
|
dict(name="B_cpu4", replicas=2, api_cpu=2.0, api_mem="512M", pg_cpu=1.0,
|
|
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE),
|
|
dict(name="B_cpu4_db2", replicas=2, api_cpu=2.0, api_mem="512M", pg_cpu=2.0,
|
|
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE),
|
|
]
|
|
|
|
|
|
def series_c():
|
|
base = dict(replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0,
|
|
pool=10, overflow=20, pool_timeout=30)
|
|
return [
|
|
dict(name="C_before", **base, nginx=NGINX_BASELINE),
|
|
dict(name="C_after", **base, nginx=NGINX_KEEPALIVE),
|
|
]
|
|
|
|
|
|
def series_d():
|
|
base = dict(replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0,
|
|
nginx=NGINX_KEEPALIVE)
|
|
return [
|
|
dict(name="D_pool_starved", **base, pool=2, overflow=0, pool_timeout=5),
|
|
dict(name="D_pool_normal", **base, pool=10, overflow=20, pool_timeout=30),
|
|
]
|
|
|
|
|
|
SERIES = {"A": series_a, "B": series_b, "C": series_c, "D": series_d}
|
|
|
|
|
|
def build_env(cfg):
|
|
env = os.environ.copy()
|
|
env["PYTHONUTF8"] = "1"
|
|
env.update({
|
|
"API_REPLICAS": str(cfg["replicas"]),
|
|
"API_CPU": str(cfg["api_cpu"]),
|
|
"API_MEM": cfg["api_mem"],
|
|
"PG_CPU": str(cfg["pg_cpu"]),
|
|
"DB_POOL_SIZE": str(cfg["pool"]),
|
|
"DB_MAX_OVERFLOW": str(cfg["overflow"]),
|
|
"DB_POOL_TIMEOUT": str(cfg.get("pool_timeout", 30)),
|
|
"DB_POOL_RECYCLE": str(cfg.get("pool_recycle", 3600)),
|
|
})
|
|
env.update(cfg["nginx"])
|
|
return env
|
|
|
|
|
|
def run(cmd, env, check=True, quiet=False):
|
|
return subprocess.run(
|
|
cmd, cwd=ROOT, env=env, check=check,
|
|
stdout=subprocess.DEVNULL if quiet else None,
|
|
stderr=subprocess.STDOUT if quiet else None,
|
|
)
|
|
|
|
|
|
def wait_ready(timeout=180):
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as r:
|
|
if r.status == 200:
|
|
return True
|
|
except (urllib.error.URLError, urllib.error.HTTPError, OSError, TimeoutError):
|
|
pass
|
|
time.sleep(2)
|
|
return False
|
|
|
|
|
|
def observe_instances(n=14):
|
|
seen = set()
|
|
for _ in range(n):
|
|
try:
|
|
with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as r:
|
|
v = r.headers.get("X-Instance-Id")
|
|
if v:
|
|
seen.add(v)
|
|
except Exception:
|
|
pass
|
|
return sorted(seen)
|
|
|
|
|
|
def psql(sql, env):
|
|
p = subprocess.run(
|
|
["docker", "compose", "exec", "-T", "postgres", "psql",
|
|
"-U", env.get("POSTGRES_USER", "mtusi"),
|
|
"-d", env.get("POSTGRES_DB", "mtusi-messenger"), "-At", "-c", sql],
|
|
cwd=ROOT, env=env, capture_output=True, text=True,
|
|
)
|
|
return p.stdout.strip() if p.returncode == 0 else ""
|
|
|
|
|
|
def pg_snapshot(env):
|
|
out = {
|
|
"numbackends": psql("SELECT count(*) FROM pg_stat_activity WHERE datname IS NOT NULL;", env),
|
|
"active": psql("SELECT count(*) FROM pg_stat_activity WHERE state='active';", env),
|
|
"waits": psql("SELECT coalesce(wait_event_type,'cpu')||':'||count(*) FROM pg_stat_activity "
|
|
"WHERE backend_type='client backend' GROUP BY 1 ORDER BY 2 DESC;", env),
|
|
"top_sql": psql("SELECT calls||'|'||round(total_exec_time::numeric,1)||'|'||left(query,70) "
|
|
"FROM pg_stat_statements WHERE query ILIKE '%messages%' "
|
|
"AND query NOT ILIKE '%pg_stat%' ORDER BY total_exec_time DESC LIMIT 5;", env),
|
|
}
|
|
return out
|
|
|
|
|
|
class StatsSampler:
|
|
def __init__(self, path, interval=2.0):
|
|
self.path = path
|
|
self.interval = interval
|
|
self.proc = None
|
|
self.fh = None
|
|
|
|
def __enter__(self):
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
self.fh = open(self.path, "w", encoding="utf-8")
|
|
fmt = '"{{.Name}}","{{.CPUPerc}}","{{.MemUsage}}","{{.NetIO}}","{{.BlockIO}}"'
|
|
script = (f'while true; do docker stats --no-stream --format {fmt}; '
|
|
f'echo "---SAMPLE---"; sleep {self.interval}; done')
|
|
self.proc = subprocess.Popen(["bash", "-c", script], cwd=ROOT,
|
|
stdout=self.fh, stderr=subprocess.DEVNULL)
|
|
return self
|
|
|
|
def __exit__(self, *a):
|
|
if self.proc:
|
|
self.proc.terminate()
|
|
try:
|
|
self.proc.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
self.proc.kill()
|
|
if self.fh:
|
|
self.fh.close()
|
|
|
|
|
|
def load_once(cfg, concurrency, duration, tag, env):
|
|
out = LOAD_DIR / f"{cfg['name']}_c{concurrency}_{tag}.json"
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
run([sys.executable, "-X", "utf8", "tests/load.py",
|
|
"--base-url", BASE_URL,
|
|
"--duration", str(duration),
|
|
"--concurrency", str(concurrency),
|
|
"--output", str(out)], env, check=False)
|
|
if not out.exists():
|
|
return None
|
|
return json.loads(out.read_text(encoding="utf-8"))
|
|
|
|
|
|
def apply_config(cfg, env):
|
|
print(f"\n\nCONFIG {cfg['name']}: replicas={cfg['replicas']} "
|
|
f"cpu/replica={cfg['api_cpu']} Σcpu={cfg['replicas']*cfg['api_cpu']:.1f} "
|
|
f"pg_cpu={cfg['pg_cpu']} "
|
|
f"pool={cfg['pool']}+{cfg['overflow']} "
|
|
f"(max {cfg['replicas']*(cfg['pool']+cfg['overflow'])} conns) "
|
|
f"timeout={cfg.get('pool_timeout',30)}s\n")
|
|
|
|
run(["docker", "compose", "up", "-d", "--no-deps", "--force-recreate",
|
|
"--scale", f"api={cfg['replicas']}", "api"], env, quiet=True)
|
|
run(["docker", "compose", "up", "-d", "--no-deps", "--force-recreate", "nginx"], env, quiet=True)
|
|
if not wait_ready():
|
|
raise RuntimeError(f"{cfg['name']}: /health/ready not reached")
|
|
time.sleep(5)
|
|
ids = observe_instances()
|
|
print(f"Observed X-Instance-Id ({len(ids)}): {ids}")
|
|
return ids
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--series", nargs="+", default=["A"], choices=list(SERIES))
|
|
ap.add_argument("--concurrency", nargs="+", type=int, default=CONCURRENCY)
|
|
ap.add_argument("--duration", type=int, default=DURATION)
|
|
ap.add_argument("--repeats", type=int, default=REPEATS)
|
|
args = ap.parse_args()
|
|
|
|
RAW.mkdir(parents=True, exist_ok=True)
|
|
results = {
|
|
"started_at": datetime.now(timezone.utc).isoformat(),
|
|
"base_url": BASE_URL,
|
|
"profile": {"duration_s": args.duration, "warmup_s": WARMUP,
|
|
"concurrency": args.concurrency, "repeats": args.repeats,
|
|
"tool": "tests/load.py (closed-loop, no retries)"},
|
|
"configs": [],
|
|
}
|
|
|
|
for s in args.series:
|
|
for cfg in SERIES[s]():
|
|
env = build_env(cfg)
|
|
ids = apply_config(cfg, env)
|
|
entry = {"series": s, **{k: v for k, v in cfg.items() if k != "nginx"},
|
|
"nginx_mode": "keepalive" if cfg["nginx"]["KEEPALIVE_DIRECTIVE"] else "baseline",
|
|
"observed_instance_ids": ids, "points": []}
|
|
|
|
print(f"Warmup {WARMUP}s @ c={args.concurrency[0]} ...")
|
|
load_once(cfg, args.concurrency[0], WARMUP, "warmup", env)
|
|
|
|
for c in args.concurrency:
|
|
runs = []
|
|
for r in range(args.repeats):
|
|
pg_before = pg_snapshot(env)
|
|
with StatsSampler(STATS_DIR / f"{cfg['name']}_c{c}_r{r}.csv"):
|
|
m = load_once(cfg, c, args.duration, f"r{r}", env)
|
|
pg_after = pg_snapshot(env)
|
|
if m is None:
|
|
print(f" c={c} r={r}: LOAD FAILED")
|
|
continue
|
|
runs.append(m)
|
|
print(f" c={c} r={r}: rps={m['successful_rps']} "
|
|
f"err={m['error_ratio']} p95={m['successful_latency_ms']['p95']}ms "
|
|
f"pg_backends={pg_after['numbackends']}")
|
|
entry["points"].append({
|
|
"concurrency": c, "repeat": r, "metrics": m,
|
|
"pg_before": pg_before, "pg_after": pg_after,
|
|
"stats_csv": str(STATS_DIR / f"{cfg['name']}_c{c}_r{r}.csv"),
|
|
})
|
|
if runs:
|
|
rps = sorted(x["successful_rps"] for x in runs)
|
|
entry.setdefault("summary", {})[str(c)] = {
|
|
"median_rps": rps[len(rps)//2],
|
|
"runs": len(runs),
|
|
}
|
|
results["configs"].append(entry)
|
|
(RAW / "raw.json").write_text(json.dumps(results, ensure_ascii=False, indent=2),
|
|
encoding="utf-8")
|
|
|
|
results["finished_at"] = datetime.now(timezone.utc).isoformat()
|
|
(RAW / "raw.json").write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print(f"\nDONE. Raw data: {RAW/'raw.json'}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |