161 lines
10 KiB
Python
161 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Repeatable local scaling study: fixed aggregate budget, then increased budget."""
|
|
import json, os, platform, random, subprocess, sys, time, urllib.request
|
|
from statistics import median
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
|
OUT = ROOT / "evidence"
|
|
LOAD = [sys.executable, "tests/load.py", "--base-url", "http://127.0.0.1:8080", "--duration", os.getenv("BENCH_SECONDS", "30"), "--concurrency", os.getenv("BENCH_CONCURRENCY", "16")]
|
|
|
|
def run(args, env=None, check=True):
|
|
subprocess.run(args, cwd=ROOT, env=env, check=check)
|
|
|
|
def docker_usage():
|
|
ids=[]
|
|
for service in ("api","postgres","haproxy"):
|
|
found=subprocess.run(COMPOSE+["ps","-q",service],cwd=ROOT,text=True,capture_output=True,check=True).stdout.split()
|
|
ids.extend(found)
|
|
if not ids: return {}
|
|
raw=subprocess.run(["docker","stats","--no-stream","--format","{{json .}}",*ids],cwd=ROOT,text=True,capture_output=True,check=True).stdout
|
|
grouped={"api":{"cpu_percent":[],"memory_mib":[]},"postgres":{"cpu_percent":[],"memory_mib":[]},"haproxy":{"cpu_percent":[],"memory_mib":[]}}
|
|
for line in raw.splitlines():
|
|
try: item=json.loads(line)
|
|
except ValueError: continue
|
|
name=item.get("Name","")
|
|
role="api" if "-api-" in name else "postgres" if "-postgres-" in name else "haproxy" if "-haproxy-" in name else None
|
|
if not role: continue
|
|
try: grouped[role]["cpu_percent"].append(float(item["CPUPerc"].rstrip("%")))
|
|
except (KeyError,ValueError): pass
|
|
try:
|
|
size=item["MemUsage"].split("/")[0].strip()
|
|
if size.endswith("GiB"): value=float(size[:-3])*1024
|
|
elif size.endswith("MiB"): value=float(size[:-3])
|
|
elif size.endswith("KiB"): value=float(size[:-3])/1024
|
|
else: value=float(size[:-1])/(1024*1024)
|
|
grouped[role]["memory_mib"].append(value)
|
|
except (KeyError,ValueError,IndexError): pass
|
|
return grouped
|
|
|
|
def row_count():
|
|
env={}
|
|
for line in (ROOT/".env").read_text().splitlines():
|
|
if "=" in line:
|
|
k,v=line.split("=",1); env[k]=v
|
|
cmd=COMPOSE+["exec","-T","postgres","psql","-U",env.get("POSTGRES_USER","messenger"),"-d",env.get("POSTGRES_DB","messenger"),"-Atc","SELECT count(*) FROM messages"]
|
|
result=subprocess.run(cmd,cwd=ROOT,text=True,capture_output=True,check=True)
|
|
return int(result.stdout.strip())
|
|
|
|
def reset_synthetic_messages():
|
|
"""Use an identical empty message/idempotency dataset for every sample."""
|
|
env={}
|
|
for line in (ROOT/".env").read_text().splitlines():
|
|
if "=" in line:
|
|
k,v=line.split("=",1); env[k]=v
|
|
cmd=COMPOSE+["exec","-T","postgres","psql","-U",env.get("POSTGRES_USER","messenger"),"-d",env.get("POSTGRES_DB","messenger"),"-v","ON_ERROR_STOP=1","-c","TRUNCATE TABLE messages, idempotency_keys; ANALYZE messages; ANALYZE idempotency_keys;"]
|
|
subprocess.run(cmd,cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,check=True)
|
|
|
|
def benchmark_load(dest, env):
|
|
reset_synthetic_messages()
|
|
before=row_count(); samples=[]
|
|
proc=subprocess.Popen(LOAD+["--output",str(dest)],cwd=ROOT,env=env,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,text=True)
|
|
while proc.poll() is None:
|
|
stats=docker_usage()
|
|
if stats: samples.append(stats)
|
|
time.sleep(1)
|
|
output=proc.communicate()[0]
|
|
print(output,end="")
|
|
if proc.returncode: raise subprocess.CalledProcessError(proc.returncode,LOAD)
|
|
after=row_count()
|
|
flat={}
|
|
for role in ("api","postgres","haproxy"):
|
|
cpus=[x for sample in samples for x in sample[role]["cpu_percent"]]
|
|
mems=[x for sample in samples for x in sample[role]["memory_mib"]]
|
|
flat[role]={"cpu_percent_mean":round(sum(cpus)/len(cpus),2) if cpus else None,"cpu_percent_peak":round(max(cpus),2) if cpus else None,"memory_mib_peak":round(max(mems),2) if mems else None}
|
|
return {"message_rows_before":before,"message_rows_after":after,"docker_stats_samples":len(samples),"container_resource_usage":flat}
|
|
|
|
def wait_ready(timeout=90):
|
|
deadline=time.monotonic()+timeout
|
|
while time.monotonic()<deadline:
|
|
try:
|
|
with urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=1) as r:
|
|
if r.status==200: return
|
|
except Exception: pass
|
|
time.sleep(.25)
|
|
raise RuntimeError("LB did not become ready in 90 seconds")
|
|
|
|
def variant(name, replicas, cpus, memory, db_pool=10, sample=1):
|
|
env=os.environ.copy(); env.update(API_CPUS=str(cpus), API_MEMORY=memory, DB_POOL_MAX=str(db_pool))
|
|
run(COMPOSE+["up","-d","--build","--scale",f"api={replicas}","--wait","--wait-timeout","180"],env)
|
|
wait_ready()
|
|
time.sleep(5) # identical settling interval after topology change
|
|
dest=OUT/(name+f"-sample{sample}.json")
|
|
stats=benchmark_load(dest,env)
|
|
item=json.loads(dest.read_text()); item.update({"variant":name,"replicas":replicas,"aggregate_api_cpu_limit":round(replicas*cpus,3),"api_memory_limit_per_replica":memory,"db_pool_max_per_replica":db_pool,**stats})
|
|
dest.write_text(json.dumps(item,ensure_ascii=False,indent=2)+"\n")
|
|
return item
|
|
|
|
def main():
|
|
OUT.mkdir(exist_ok=True)
|
|
samples=[]
|
|
repeats=int(os.getenv("BENCH_REPEATS", "3"))
|
|
cases=[]
|
|
# Two vCPU / 1 GiB total API budget, distributed equally across replicas.
|
|
cases += [(f"fixed-{n}",n,c,m,10) for n,c,m in [(1,2,"1g"),(2,1,"512m"),(4,.5,"256m")]]
|
|
# More total budget at 0.5 CPU / 256 MiB per replica.
|
|
cases += [(f"scaled-{n}",n,.5,"256m",10) for n in (1,2,4,8)]
|
|
# Tuning comparison at identical four-replica CPU/RAM to find a supported improvement.
|
|
cases += [(f"pool-{pool}",4,.5,"256m",pool) for pool in (5,10,20)]
|
|
rng=random.Random(20261105)
|
|
for sample in range(1,repeats+1):
|
|
block=cases[:]; rng.shuffle(block)
|
|
for name,n,c,m,pool in block:
|
|
row=variant(name,n,c,m,pool,sample)
|
|
row["sample"]=sample; row["db_pool_max_per_replica"]=pool
|
|
samples.append(row)
|
|
medians=[]
|
|
for name,n,c,m,pool in cases:
|
|
group=[r for r in samples if r["variant"]==name]
|
|
combined={"variant":name,"replicas":n,"aggregate_api_cpu_limit":round(n*c,3),"api_memory_limit_per_replica":m,"db_pool_max_per_replica":pool}
|
|
for metric in ("successful_rps",): combined[metric]=round(median(r[metric] for r in group),3)
|
|
for q in ("p50","p95","p99"):
|
|
combined.setdefault("successful_latency_ms",{})[q]=round(median(r["successful_latency_ms"][q] for r in group),3)
|
|
combined["error_ratio"]=round(median(r["error_ratio"] for r in group),6)
|
|
combined["successful_requests"]=int(median(r["successful_requests"] for r in group))
|
|
combined["failed_requests"]=int(median(r["failed_requests"] for r in group))
|
|
combined["message_rows_before"]=int(median(r["message_rows_before"] for r in group))
|
|
combined["message_rows_after"]=int(median(r["message_rows_after"] for r in group))
|
|
combined["container_resource_usage"]={}
|
|
for role in ("api","postgres","haproxy"):
|
|
combined["container_resource_usage"][role]={key:round(median(r["container_resource_usage"][role][key] for r in group if r["container_resource_usage"][role][key] is not None),2) if any(r["container_resource_usage"][role][key] is not None for r in group) else None for key in ("cpu_percent_mean","cpu_percent_peak","memory_mib_peak")}
|
|
medians.append(combined)
|
|
payload={"method":"closed-loop POST, 5s stabilization after each topology change; each sample starts after TRUNCATE of synthetic messages/idempotency_keys and ANALYZE; deterministic randomized blocks; median summary plus every raw sample","duration_seconds":float(os.getenv("BENCH_SECONDS","30")),"concurrency":int(os.getenv("BENCH_CONCURRENCY","16")),"repeat_count":repeats,"random_seed":20261105,"host_platform":platform.platform(),"host_cpu_count":os.cpu_count(),"docker_server_version":subprocess.run(["docker","version","--format","{{.Server.Version}}"],cwd=ROOT,text=True,capture_output=True,check=True).stdout.strip(),"results":medians,"samples":samples}
|
|
(OUT/"track1-results.json").write_text(json.dumps(payload,ensure_ascii=False,indent=2)+"\n")
|
|
make_svg([r for r in medians if r["variant"].startswith(("fixed-","scaled-"))],OUT/"track1-rps.svg")
|
|
print(OUT/"track1-results.json")
|
|
|
|
def make_svg(rows,path):
|
|
# Small dependency-free chart for the report; data values remain in JSON.
|
|
groups=[("Fixed aggregate budget",[r for r in rows if r["variant"].startswith("fixed-")]),("Increased aggregate budget",[r for r in rows if r["variant"].startswith("scaled-")])]
|
|
w,h=900,460; left,top,plotw,ploth=90,55,740,310
|
|
vmax=max((r["successful_rps"] for r in rows),default=1) or 1
|
|
out=[f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" viewBox="0 0 {w} {h}">','<rect width="100%" height="100%" fill="white"/>','<text x="450" y="28" text-anchor="middle" font-family="sans-serif" font-size="20">API scaling: successful requests per second</text>']
|
|
for tick in range(5):
|
|
y=top+ploth-tick*ploth/4; val=vmax*tick/4
|
|
out += [f'<line x1="{left}" y1="{y:.1f}" x2="{left+plotw}" y2="{y:.1f}" stroke="#ddd"/>',f'<text x="{left-10}" y="{y+4:.1f}" text-anchor="end" font-family="sans-serif" font-size="12">{val:.1f}</text>']
|
|
colors=["#2563eb","#f97316"]
|
|
for gi,(label,items) in enumerate(groups):
|
|
pts=[]
|
|
for i,r in enumerate(items):
|
|
x=left+plotw*({1:1,2:2,4:3,8:4}[r["replicas"]])/5; y=top+ploth-(r["successful_rps"]/vmax)*ploth
|
|
pts.append(f"{x:.1f},{y:.1f}"); out.append(f'<circle cx="{x:.1f}" cy="{y:.1f}" r="6" fill="{colors[gi]}"/>'); out.append(f'<text x="{x:.1f}" y="{y-10:.1f}" text-anchor="middle" font-family="sans-serif" font-size="12">{r["successful_rps"]:.1f}</text>')
|
|
out.append(f'<polyline points="{" ".join(pts)}" fill="none" stroke="{colors[gi]}" stroke-width="3"/>')
|
|
positions={1:1,2:2,4:3,8:4}
|
|
for n,pos in positions.items():
|
|
x=left+plotw*pos/5; out.append(f'<text x="{x:.1f}" y="{top+ploth+25}" text-anchor="middle" font-family="sans-serif" font-size="14">{n} replicas</text>')
|
|
out += [f'<line x1="130" y1="420" x2="160" y2="420" stroke="{colors[0]}" stroke-width="3"/><text x="168" y="424" font-family="sans-serif" font-size="13">{groups[0][0]}</text>',f'<line x1="430" y1="420" x2="460" y2="420" stroke="{colors[1]}" stroke-width="3"/><text x="468" y="424" font-family="sans-serif" font-size="13">{groups[1][0]}</text>','</svg>']
|
|
path.write_text("\n".join(out)+"\n")
|
|
|
|
if __name__=="__main__": main()
|