feat-(overloaded pool 503 added)

This commit is contained in:
2026-11-17 16:34:35 +03:00
parent 9af1020087
commit 036fe83c4d
14 changed files with 699 additions and 43 deletions
+161
View File
@@ -0,0 +1,161 @@
import argparse
import base64
import json
import statistics
import sys
import time
import urllib.error
import urllib.request
import uuid
from concurrent.futures import ThreadPoolExecutor
BASE = "http://localhost:8080"
def setup():
pwd = "SecurePassword123!"
ids = []
for _ in range(2):
u = f"ra_{uuid.uuid4().hex[:10]}"
s, _, user, _ = call("POST", "/api/v1/users",
body={"username": u, "display_name": "RA", "password": pwd})
assert s == 201, (s, user)
ids.append((u, user["id"]))
cred = base64.b64encode(f"{ids[0][0]}:{pwd}".encode()).decode()
s, _, sess, _ = call("POST", "/api/v1/auth/sessions",
headers={"Authorization": f"Basic {cred}"})
assert s == 201, (s, sess)
h = {"Authorization": f"Bearer {sess['token']}"}
s, _, chat, _ = call("POST", "/api/v1/chats", headers=h,
body={"title": "Retry-After check", "member_ids": [i[1] for i in ids]})
assert s == 201, (s, chat)
return h, chat["id"]
def pct(vals, p):
if not vals:
return None
v = sorted(vals)
return round(1000 * v[min(len(v) - 1, int(p * len(v)))], 1)
def call(method, path, headers=None, body=None):
data = json.dumps(body).encode() if body is not None else None
req = urllib.request.Request(BASE + path, data=data, method=method)
req.add_header("Accept", "application/json")
if data:
req.add_header("Content-Type", "application/json")
for k, v in (headers or {}).items():
req.add_header(k, v)
t0 = time.monotonic()
try:
with urllib.request.urlopen(req, timeout=60) as r:
raw = r.read().decode()
hdr = {k.lower(): v for k, v in r.headers.items()}
return r.status, hdr, json.loads(raw) if raw else None, time.monotonic() - t0
except urllib.error.HTTPError as e:
raw = e.read().decode()
hdr = {k.lower(): v for k, v in e.headers.items()}
try:
parsed = json.loads(raw) if raw else None
except ValueError:
parsed = {"_raw": raw[:200]}
return e.code, hdr, parsed, time.monotonic() - t0
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--n", type=int, default=64)
ap.add_argument("--allow-zero-503", action="store_true")
args = ap.parse_args()
n = args.n
h, chat_id = setup()
keys = [f"ra-{uuid.uuid4().hex}" for _ in range(n)]
bodies = [{"text": f"retry-after probe {i}"} for i in range(n)]
def send(i):
s, hdr, body, dt = call("POST", f"/api/v1/chats/{chat_id}/messages",
headers={**h, "Idempotency-Key": keys[i]}, body=bodies[i])
return i, s, hdr, body, dt
wall0 = time.monotonic()
with ThreadPoolExecutor(max_workers=n) as ex:
first = list(ex.map(send, range(n)))
wall = time.monotonic() - wall0
codes, problems, retry_idx, lat = {}, [], [], []
instances = set()
for i, s, hdr, body, dt in first:
codes[s] = codes.get(s, 0) + 1
lat.append(dt)
if hdr.get("x-instance-id"):
instances.add(hdr["x-instance-id"])
if s == 503:
retry_idx.append(i)
ra = hdr.get("retry-after")
err = (body or {}).get("error", {})
if err.get("code") != "unavailable":
problems.append(f"[{i}] code={err.get('code')}, ожидался unavailable")
if not ra or not ra.isdigit() or int(ra) < 1:
problems.append(f"[{i}] некорректный Retry-After: {ra!r}")
if err.get("request_id") != hdr.get("x-request-id"):
problems.append(f"[{i}] request_id в теле != заголовку")
if not hdr.get("x-instance-id"):
problems.append(f"[{i}] нет X-Instance-Id")
if "QueuePool" in json.dumps(body):
problems.append(f"[{i}] утечка внутренних деталей")
if s == 500:
problems.append(f"[{i}] 500 instead of 503: {body}")
print(f"Concurrent: {n}; wall time: {wall:.2f}s; replices found: {len(instances)} {sorted(instances)}")
print(f"Statusues of 1 cycle: {codes}")
print(f"Latency, ms: p50={pct(lat,.5)} p95={pct(lat,.95)} p99={pct(lat,.99)}")
print(f"503 с Retry-After: {len(retry_idx)}")
if retry_idx:
smp = next(r for r in first if r[1] == 503)
print(f"Example: Retry-After={smp[2].get('retry-after')}, "
f"message={smp[3]['error']['message']!r}")
ok_ids = {b["id"] for _, s, _, b, _ in first if s == 201}
for i in retry_idx:
s, hdr, body, _ = call("POST", f"/api/v1/chats/{chat_id}/messages",
headers={**h, "Idempotency-Key": keys[i]}, body=bodies[i])
if s != 201:
problems.append(f"retry[{i}] -> {s}, expected 201: {body}")
else:
ok_ids.add(body["id"])
total, cursor = 0, None
while True:
p = f"/api/v1/chats/{chat_id}/messages?limit=100"
if cursor:
p += "&cursor=" + cursor
s, _, page, _ = call("GET", p, headers=h)
assert s == 200, (s, page)
total += len(page["items"])
cursor = page["next_cursor"]
if cursor is None:
break
print(f"Messages in chat: {total}; unique id: {len(ok_ids)}; keys: {n}")
if total != n or len(ok_ids) != n:
problems.append(f"dublicates/losses: total={total}, ids={len(ok_ids)}, expected {n}")
if problems:
print("\nFAIL:")
for p in problems[:15]:
print(" -", p)
if len(problems) > 15:
print(f" ... и ещё {len(problems)-15}")
sys.exit(1)
if not retry_idx and not args.allow_zero_503:
print("\nINCONCLUSIVE: 503 не наблюдалось — маппинг пула НЕ проверен.")
sys.exit(2)
print("\nPASS: 503 unavailable + Retry-After correct, retries are safe, no dublicates")
if __name__ == "__main__":
main()
+75
View File
@@ -0,0 +1,75 @@
import json
import statistics
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
ROOT = Path(__file__).resolve().parents[1]
RAW = ROOT / "results" / "scaling" / "raw.json"
OUT = ROOT / "results" / "scaling"
OUT.mkdir(parents=True, exist_ok=True)
data = json.loads(RAW.read_text(encoding="utf-8"))
def points(cfg, key):
agg = {}
for p in cfg["points"]:
agg.setdefault(p["concurrency"], []).append(p["metrics"])
xs = sorted(agg)
ys = []
for x in xs:
vals = [m[key] if key in m else m["successful_latency_ms"][key] for m in agg[x]]
ys.append(statistics.median(vals))
return xs, ys
def plot_series(series, fname, title):
cfgs = [c for c in data["configs"] if c["series"] == series]
if not cfgs:
return
fig, ax = plt.subplots(1, 3, figsize=(16, 4.2))
for c in cfgs:
label = f"{c['name']} (r={c['replicas']}, Σcpu={c['replicas']*c['api_cpu']:.1f})"
x, y = points(c, "successful_rps"); ax[0].plot(x, y, "o-", label=label)
x, y = points(c, "p95"); ax[1].plot(x, y, "o-", label=label)
x, y = points(c, "error_ratio"); ax[2].plot(x, y, "o-", label=label)
ax[0].set(xlabel="concurrency", ylabel="successful RPS", title="Throughput")
ax[1].set(xlabel="concurrency", ylabel="p95 latency, ms", title="Tail latency")
ax[2].set(xlabel="concurrency", ylabel="error ratio", title="Errors")
for a in ax:
a.grid(alpha=.3); a.legend(fontsize=7)
fig.suptitle(title)
fig.tight_layout()
fig.savefig(OUT / fname, dpi=140)
print("saved", OUT / fname)
plot_series("A", "serie_a_fixed_budget.png",
"Serie A: fixed total API budget (2.0 CPU / 1024 MiB), 1 vs 2 vs 4 replicas")
plot_series("B", "serie_b_resource_growth.png",
"Serie B: growing API CPU, Postgres fixed")
plot_series("C", "serie_c_keepalive_ab.png",
"Serie C: nginx upstream keepalive A/B (identical resources)")
plot_series("D", "serie_d_pool_saturation.png",
"Serie D: connection-pool saturation at identical CPU (2x1.0)")
lines = ["| config | replicas | Σcpu API | conc | median RPS | p50 | p95 | p99 | err |",
"|---|---|---|---|---|---|---|---|---|"]
for c in data["configs"]:
agg = {}
for p in c["points"]:
agg.setdefault(p["concurrency"], []).append(p["metrics"])
for conc in sorted(agg):
ms = agg[conc]
med = lambda f: round(statistics.median([f(m) for m in ms]), 3)
lines.append(
f"| {c['name']} | {c['replicas']} | {c['replicas']*c['api_cpu']:.1f} | {conc} | "
f"{med(lambda m: m['successful_rps'])} | "
f"{med(lambda m: m['successful_latency_ms']['p50'])} | "
f"{med(lambda m: m['successful_latency_ms']['p95'])} | "
f"{med(lambda m: m['successful_latency_ms']['p99'])} | "
f"{med(lambda m: m['error_ratio'])} |")
(OUT / "summary_table.md").write_text("\n".join(lines), encoding="utf-8")
print("saved", OUT / "summary_table.md")
+278
View File
@@ -0,0 +1,278 @@
import argparse
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
RAW = ROOT / "results" / "scaling"
LOAD_DIR = RAW / "load"
STATS_DIR = RAW / "stats"
BASE_URL = os.getenv("BENCH_BASE_URL", "http://localhost:8080")
CONCURRENCY = [8, 16, 32, 64]
DURATION = 30
WARMUP = 10
REPEATS = 2
NGINX_BASELINE = {
"KEEPALIVE_DIRECTIVE": "",
"PROXY_HTTP_VERSION_DIRECTIVE": "",
"PROXY_CONNECTION_DIRECTIVE": "",
}
NGINX_KEEPALIVE = {
"KEEPALIVE_DIRECTIVE": "keepalive 64;",
"PROXY_HTTP_VERSION_DIRECTIVE": "proxy_http_version 1.1;",
"PROXY_CONNECTION_DIRECTIVE": 'proxy_set_header Connection "";',
}
def series_a():
return [
dict(name="A_r1", replicas=1, api_cpu=2.0, api_mem="1024M", pg_cpu=1.0,
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_BASELINE),
dict(name="A_r2", replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0,
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_BASELINE),
dict(name="A_r4", replicas=4, api_cpu=0.5, api_mem="256M", pg_cpu=1.0,
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_BASELINE),
]
def series_b():
return [
dict(name="B_cpu1", replicas=2, api_cpu=0.5, api_mem="512M", pg_cpu=1.0,
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE),
dict(name="B_cpu2", replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0,
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE),
dict(name="B_cpu4", replicas=2, api_cpu=2.0, api_mem="512M", pg_cpu=1.0,
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE),
dict(name="B_cpu4_db2", replicas=2, api_cpu=2.0, api_mem="512M", pg_cpu=2.0,
pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE),
]
def series_c():
base = dict(replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0,
pool=10, overflow=20, pool_timeout=30)
return [
dict(name="C_before", **base, nginx=NGINX_BASELINE),
dict(name="C_after", **base, nginx=NGINX_KEEPALIVE),
]
def series_d():
base = dict(replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0,
nginx=NGINX_KEEPALIVE)
return [
dict(name="D_pool_starved", **base, pool=2, overflow=0, pool_timeout=5),
dict(name="D_pool_normal", **base, pool=10, overflow=20, pool_timeout=30),
]
SERIES = {"A": series_a, "B": series_b, "C": series_c, "D": series_d}
def build_env(cfg):
env = os.environ.copy()
env["PYTHONUTF8"] = "1"
env.update({
"API_REPLICAS": str(cfg["replicas"]),
"API_CPU": str(cfg["api_cpu"]),
"API_MEM": cfg["api_mem"],
"PG_CPU": str(cfg["pg_cpu"]),
"DB_POOL_SIZE": str(cfg["pool"]),
"DB_MAX_OVERFLOW": str(cfg["overflow"]),
"DB_POOL_TIMEOUT": str(cfg.get("pool_timeout", 30)),
"DB_POOL_RECYCLE": str(cfg.get("pool_recycle", 3600)),
})
env.update(cfg["nginx"])
return env
def run(cmd, env, check=True, quiet=False):
return subprocess.run(
cmd, cwd=ROOT, env=env, check=check,
stdout=subprocess.DEVNULL if quiet else None,
stderr=subprocess.STDOUT if quiet else None,
)
def wait_ready(timeout=180):
deadline = time.time() + timeout
while time.time() < deadline:
try:
with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as r:
if r.status == 200:
return True
except (urllib.error.URLError, urllib.error.HTTPError, OSError, TimeoutError):
pass
time.sleep(2)
return False
def observe_instances(n=14):
seen = set()
for _ in range(n):
try:
with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as r:
v = r.headers.get("X-Instance-Id")
if v:
seen.add(v)
except Exception:
pass
return sorted(seen)
def psql(sql, env):
p = subprocess.run(
["docker", "compose", "exec", "-T", "postgres", "psql",
"-U", env.get("POSTGRES_USER", "mtusi"),
"-d", env.get("POSTGRES_DB", "mtusi-messenger"), "-At", "-c", sql],
cwd=ROOT, env=env, capture_output=True, text=True,
)
return p.stdout.strip() if p.returncode == 0 else ""
def pg_snapshot(env):
out = {
"numbackends": psql("SELECT count(*) FROM pg_stat_activity WHERE datname IS NOT NULL;", env),
"active": psql("SELECT count(*) FROM pg_stat_activity WHERE state='active';", env),
"waits": psql("SELECT coalesce(wait_event_type,'cpu')||':'||count(*) FROM pg_stat_activity "
"WHERE backend_type='client backend' GROUP BY 1 ORDER BY 2 DESC;", env),
"top_sql": psql("SELECT calls||'|'||round(total_exec_time::numeric,1)||'|'||left(query,70) "
"FROM pg_stat_statements WHERE query ILIKE '%messages%' "
"AND query NOT ILIKE '%pg_stat%' ORDER BY total_exec_time DESC LIMIT 5;", env),
}
return out
class StatsSampler:
def __init__(self, path, interval=2.0):
self.path = path
self.interval = interval
self.proc = None
self.fh = None
def __enter__(self):
self.path.parent.mkdir(parents=True, exist_ok=True)
self.fh = open(self.path, "w", encoding="utf-8")
fmt = '"{{.Name}}","{{.CPUPerc}}","{{.MemUsage}}","{{.NetIO}}","{{.BlockIO}}"'
script = (f'while true; do docker stats --no-stream --format {fmt}; '
f'echo "---SAMPLE---"; sleep {self.interval}; done')
self.proc = subprocess.Popen(["bash", "-c", script], cwd=ROOT,
stdout=self.fh, stderr=subprocess.DEVNULL)
return self
def __exit__(self, *a):
if self.proc:
self.proc.terminate()
try:
self.proc.wait(timeout=5)
except subprocess.TimeoutExpired:
self.proc.kill()
if self.fh:
self.fh.close()
def load_once(cfg, concurrency, duration, tag, env):
out = LOAD_DIR / f"{cfg['name']}_c{concurrency}_{tag}.json"
out.parent.mkdir(parents=True, exist_ok=True)
run([sys.executable, "-X", "utf8", "tests/load.py",
"--base-url", BASE_URL,
"--duration", str(duration),
"--concurrency", str(concurrency),
"--output", str(out)], env, check=False)
if not out.exists():
return None
return json.loads(out.read_text(encoding="utf-8"))
def apply_config(cfg, env):
print(f"\n\nCONFIG {cfg['name']}: replicas={cfg['replicas']} "
f"cpu/replica={cfg['api_cpu']} Σcpu={cfg['replicas']*cfg['api_cpu']:.1f} "
f"pg_cpu={cfg['pg_cpu']} "
f"pool={cfg['pool']}+{cfg['overflow']} "
f"(max {cfg['replicas']*(cfg['pool']+cfg['overflow'])} conns) "
f"timeout={cfg.get('pool_timeout',30)}s\n")
run(["docker", "compose", "up", "-d", "--no-deps", "--force-recreate",
"--scale", f"api={cfg['replicas']}", "api"], env, quiet=True)
run(["docker", "compose", "up", "-d", "--no-deps", "--force-recreate", "nginx"], env, quiet=True)
if not wait_ready():
raise RuntimeError(f"{cfg['name']}: /health/ready not reached")
time.sleep(5)
ids = observe_instances()
print(f"Observed X-Instance-Id ({len(ids)}): {ids}")
return ids
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--series", nargs="+", default=["A"], choices=list(SERIES))
ap.add_argument("--concurrency", nargs="+", type=int, default=CONCURRENCY)
ap.add_argument("--duration", type=int, default=DURATION)
ap.add_argument("--repeats", type=int, default=REPEATS)
args = ap.parse_args()
RAW.mkdir(parents=True, exist_ok=True)
results = {
"started_at": datetime.now(timezone.utc).isoformat(),
"base_url": BASE_URL,
"profile": {"duration_s": args.duration, "warmup_s": WARMUP,
"concurrency": args.concurrency, "repeats": args.repeats,
"tool": "tests/load.py (closed-loop, no retries)"},
"configs": [],
}
for s in args.series:
for cfg in SERIES[s]():
env = build_env(cfg)
ids = apply_config(cfg, env)
entry = {"series": s, **{k: v for k, v in cfg.items() if k != "nginx"},
"nginx_mode": "keepalive" if cfg["nginx"]["KEEPALIVE_DIRECTIVE"] else "baseline",
"observed_instance_ids": ids, "points": []}
print(f"Warmup {WARMUP}s @ c={args.concurrency[0]} ...")
load_once(cfg, args.concurrency[0], WARMUP, "warmup", env)
for c in args.concurrency:
runs = []
for r in range(args.repeats):
pg_before = pg_snapshot(env)
with StatsSampler(STATS_DIR / f"{cfg['name']}_c{c}_r{r}.csv"):
m = load_once(cfg, c, args.duration, f"r{r}", env)
pg_after = pg_snapshot(env)
if m is None:
print(f" c={c} r={r}: LOAD FAILED")
continue
runs.append(m)
print(f" c={c} r={r}: rps={m['successful_rps']} "
f"err={m['error_ratio']} p95={m['successful_latency_ms']['p95']}ms "
f"pg_backends={pg_after['numbackends']}")
entry["points"].append({
"concurrency": c, "repeat": r, "metrics": m,
"pg_before": pg_before, "pg_after": pg_after,
"stats_csv": str(STATS_DIR / f"{cfg['name']}_c{c}_r{r}.csv"),
})
if runs:
rps = sorted(x["successful_rps"] for x in runs)
entry.setdefault("summary", {})[str(c)] = {
"median_rps": rps[len(rps)//2],
"runs": len(runs),
}
results["configs"].append(entry)
(RAW / "raw.json").write_text(json.dumps(results, ensure_ascii=False, indent=2),
encoding="utf-8")
results["finished_at"] = datetime.now(timezone.utc).isoformat()
(RAW / "raw.json").write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
print(f"\nDONE. Raw data: {RAW/'raw.json'}")
if __name__ == "__main__":
main()