Добавить scripts отказа и трека масштабирования

This commit is contained in:
2026-11-06 19:12:34 +03:00
parent 77dae0ec87
commit 41edfbc66b
5 changed files with 391 additions and 0 deletions
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
# Restart every stateful dependency and all API processes without deleting volumes.
docker compose --env-file .env restart api haproxy postgres valkey
+25
View File
@@ -0,0 +1,25 @@
#!/bin/sh
set -eu
compose="docker compose --env-file .env"
# Add one ready replica before removing any old container.
$compose up -d --scale api=3 --wait --wait-timeout 120 api
python3 - <<'PY'
import time, urllib.request
end=time.monotonic()+60
while time.monotonic()<end:
try:
urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=1).read()
break
except Exception:
time.sleep(.25)
else:
raise SystemExit('LB did not become ready')
PY
# Remove one old instance at a time while another healthy backend is present.
for id in $(docker compose --env-file .env ps -q api | head -n 2); do
docker stop --time 3 "$id" >/dev/null
docker rm "$id" >/dev/null
docker compose --env-file .env up -d --scale api=3 --wait --wait-timeout 120 api
sleep 3
done
docker compose --env-file .env up -d --scale api=2 --wait --wait-timeout 120 api
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
# X-Instance-Id is the container hostname (the Docker short ID). Resolve it to
# exactly one scaled Compose container and stop only that process.
for id in $(docker compose --env-file .env ps -q api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id")
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
docker stop --time 3 "$id" >/dev/null
exit 0
fi
done
echo "No running api container matches TARGET_INSTANCE_ID" >&2
exit 1
+160
View File
@@ -0,0 +1,160 @@
#!/usr/bin/env python3
"""Repeatable local scaling study: fixed aggregate budget, then increased budget."""
import json, os, platform, random, subprocess, sys, time, urllib.request
from statistics import median
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
COMPOSE = ["docker", "compose", "--env-file", ".env"]
OUT = ROOT / "evidence"
LOAD = [sys.executable, "tests/load.py", "--base-url", "http://127.0.0.1:8080", "--duration", os.getenv("BENCH_SECONDS", "30"), "--concurrency", os.getenv("BENCH_CONCURRENCY", "16")]
def run(args, env=None, check=True):
subprocess.run(args, cwd=ROOT, env=env, check=check)
def docker_usage():
ids=[]
for service in ("api","postgres","haproxy"):
found=subprocess.run(COMPOSE+["ps","-q",service],cwd=ROOT,text=True,capture_output=True,check=True).stdout.split()
ids.extend(found)
if not ids: return {}
raw=subprocess.run(["docker","stats","--no-stream","--format","{{json .}}",*ids],cwd=ROOT,text=True,capture_output=True,check=True).stdout
grouped={"api":{"cpu_percent":[],"memory_mib":[]},"postgres":{"cpu_percent":[],"memory_mib":[]},"haproxy":{"cpu_percent":[],"memory_mib":[]}}
for line in raw.splitlines():
try: item=json.loads(line)
except ValueError: continue
name=item.get("Name","")
role="api" if "-api-" in name else "postgres" if "-postgres-" in name else "haproxy" if "-haproxy-" in name else None
if not role: continue
try: grouped[role]["cpu_percent"].append(float(item["CPUPerc"].rstrip("%")))
except (KeyError,ValueError): pass
try:
size=item["MemUsage"].split("/")[0].strip()
if size.endswith("GiB"): value=float(size[:-3])*1024
elif size.endswith("MiB"): value=float(size[:-3])
elif size.endswith("KiB"): value=float(size[:-3])/1024
else: value=float(size[:-1])/(1024*1024)
grouped[role]["memory_mib"].append(value)
except (KeyError,ValueError,IndexError): pass
return grouped
def row_count():
env={}
for line in (ROOT/".env").read_text().splitlines():
if "=" in line:
k,v=line.split("=",1); env[k]=v
cmd=COMPOSE+["exec","-T","postgres","psql","-U",env.get("POSTGRES_USER","messenger"),"-d",env.get("POSTGRES_DB","messenger"),"-Atc","SELECT count(*) FROM messages"]
result=subprocess.run(cmd,cwd=ROOT,text=True,capture_output=True,check=True)
return int(result.stdout.strip())
def reset_synthetic_messages():
"""Use an identical empty message/idempotency dataset for every sample."""
env={}
for line in (ROOT/".env").read_text().splitlines():
if "=" in line:
k,v=line.split("=",1); env[k]=v
cmd=COMPOSE+["exec","-T","postgres","psql","-U",env.get("POSTGRES_USER","messenger"),"-d",env.get("POSTGRES_DB","messenger"),"-v","ON_ERROR_STOP=1","-c","TRUNCATE TABLE messages, idempotency_keys; ANALYZE messages; ANALYZE idempotency_keys;"]
subprocess.run(cmd,cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,check=True)
def benchmark_load(dest, env):
reset_synthetic_messages()
before=row_count(); samples=[]
proc=subprocess.Popen(LOAD+["--output",str(dest)],cwd=ROOT,env=env,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,text=True)
while proc.poll() is None:
stats=docker_usage()
if stats: samples.append(stats)
time.sleep(1)
output=proc.communicate()[0]
print(output,end="")
if proc.returncode: raise subprocess.CalledProcessError(proc.returncode,LOAD)
after=row_count()
flat={}
for role in ("api","postgres","haproxy"):
cpus=[x for sample in samples for x in sample[role]["cpu_percent"]]
mems=[x for sample in samples for x in sample[role]["memory_mib"]]
flat[role]={"cpu_percent_mean":round(sum(cpus)/len(cpus),2) if cpus else None,"cpu_percent_peak":round(max(cpus),2) if cpus else None,"memory_mib_peak":round(max(mems),2) if mems else None}
return {"message_rows_before":before,"message_rows_after":after,"docker_stats_samples":len(samples),"container_resource_usage":flat}
def wait_ready(timeout=90):
deadline=time.monotonic()+timeout
while time.monotonic()<deadline:
try:
with urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=1) as r:
if r.status==200: return
except Exception: pass
time.sleep(.25)
raise RuntimeError("LB did not become ready in 90 seconds")
def variant(name, replicas, cpus, memory, db_pool=10, sample=1):
env=os.environ.copy(); env.update(API_CPUS=str(cpus), API_MEMORY=memory, DB_POOL_MAX=str(db_pool))
run(COMPOSE+["up","-d","--build","--scale",f"api={replicas}","--wait","--wait-timeout","180"],env)
wait_ready()
time.sleep(5) # identical settling interval after topology change
dest=OUT/(name+f"-sample{sample}.json")
stats=benchmark_load(dest,env)
item=json.loads(dest.read_text()); item.update({"variant":name,"replicas":replicas,"aggregate_api_cpu_limit":round(replicas*cpus,3),"api_memory_limit_per_replica":memory,"db_pool_max_per_replica":db_pool,**stats})
dest.write_text(json.dumps(item,ensure_ascii=False,indent=2)+"\n")
return item
def main():
OUT.mkdir(exist_ok=True)
samples=[]
repeats=int(os.getenv("BENCH_REPEATS", "3"))
cases=[]
# Two vCPU / 1 GiB total API budget, distributed equally across replicas.
cases += [(f"fixed-{n}",n,c,m,10) for n,c,m in [(1,2,"1g"),(2,1,"512m"),(4,.5,"256m")]]
# More total budget at 0.5 CPU / 256 MiB per replica.
cases += [(f"scaled-{n}",n,.5,"256m",10) for n in (1,2,4,8)]
# Tuning comparison at identical four-replica CPU/RAM to find a supported improvement.
cases += [(f"pool-{pool}",4,.5,"256m",pool) for pool in (5,10,20)]
rng=random.Random(20261105)
for sample in range(1,repeats+1):
block=cases[:]; rng.shuffle(block)
for name,n,c,m,pool in block:
row=variant(name,n,c,m,pool,sample)
row["sample"]=sample; row["db_pool_max_per_replica"]=pool
samples.append(row)
medians=[]
for name,n,c,m,pool in cases:
group=[r for r in samples if r["variant"]==name]
combined={"variant":name,"replicas":n,"aggregate_api_cpu_limit":round(n*c,3),"api_memory_limit_per_replica":m,"db_pool_max_per_replica":pool}
for metric in ("successful_rps",): combined[metric]=round(median(r[metric] for r in group),3)
for q in ("p50","p95","p99"):
combined.setdefault("successful_latency_ms",{})[q]=round(median(r["successful_latency_ms"][q] for r in group),3)
combined["error_ratio"]=round(median(r["error_ratio"] for r in group),6)
combined["successful_requests"]=int(median(r["successful_requests"] for r in group))
combined["failed_requests"]=int(median(r["failed_requests"] for r in group))
combined["message_rows_before"]=int(median(r["message_rows_before"] for r in group))
combined["message_rows_after"]=int(median(r["message_rows_after"] for r in group))
combined["container_resource_usage"]={}
for role in ("api","postgres","haproxy"):
combined["container_resource_usage"][role]={key:round(median(r["container_resource_usage"][role][key] for r in group if r["container_resource_usage"][role][key] is not None),2) if any(r["container_resource_usage"][role][key] is not None for r in group) else None for key in ("cpu_percent_mean","cpu_percent_peak","memory_mib_peak")}
medians.append(combined)
payload={"method":"closed-loop POST, 5s stabilization after each topology change; each sample starts after TRUNCATE of synthetic messages/idempotency_keys and ANALYZE; deterministic randomized blocks; median summary plus every raw sample","duration_seconds":float(os.getenv("BENCH_SECONDS","30")),"concurrency":int(os.getenv("BENCH_CONCURRENCY","16")),"repeat_count":repeats,"random_seed":20261105,"host_platform":platform.platform(),"host_cpu_count":os.cpu_count(),"docker_server_version":subprocess.run(["docker","version","--format","{{.Server.Version}}"],cwd=ROOT,text=True,capture_output=True,check=True).stdout.strip(),"results":medians,"samples":samples}
(OUT/"track1-results.json").write_text(json.dumps(payload,ensure_ascii=False,indent=2)+"\n")
make_svg([r for r in medians if r["variant"].startswith(("fixed-","scaled-"))],OUT/"track1-rps.svg")
print(OUT/"track1-results.json")
def make_svg(rows,path):
# Small dependency-free chart for the report; data values remain in JSON.
groups=[("Fixed aggregate budget",[r for r in rows if r["variant"].startswith("fixed-")]),("Increased aggregate budget",[r for r in rows if r["variant"].startswith("scaled-")])]
w,h=900,460; left,top,plotw,ploth=90,55,740,310
vmax=max((r["successful_rps"] for r in rows),default=1) or 1
out=[f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" viewBox="0 0 {w} {h}">','<rect width="100%" height="100%" fill="white"/>','<text x="450" y="28" text-anchor="middle" font-family="sans-serif" font-size="20">API scaling: successful requests per second</text>']
for tick in range(5):
y=top+ploth-tick*ploth/4; val=vmax*tick/4
out += [f'<line x1="{left}" y1="{y:.1f}" x2="{left+plotw}" y2="{y:.1f}" stroke="#ddd"/>',f'<text x="{left-10}" y="{y+4:.1f}" text-anchor="end" font-family="sans-serif" font-size="12">{val:.1f}</text>']
colors=["#2563eb","#f97316"]
for gi,(label,items) in enumerate(groups):
pts=[]
for i,r in enumerate(items):
x=left+plotw*({1:1,2:2,4:3,8:4}[r["replicas"]])/5; y=top+ploth-(r["successful_rps"]/vmax)*ploth
pts.append(f"{x:.1f},{y:.1f}"); out.append(f'<circle cx="{x:.1f}" cy="{y:.1f}" r="6" fill="{colors[gi]}"/>'); out.append(f'<text x="{x:.1f}" y="{y-10:.1f}" text-anchor="middle" font-family="sans-serif" font-size="12">{r["successful_rps"]:.1f}</text>')
out.append(f'<polyline points="{" ".join(pts)}" fill="none" stroke="{colors[gi]}" stroke-width="3"/>')
positions={1:1,2:2,4:3,8:4}
for n,pos in positions.items():
x=left+plotw*pos/5; out.append(f'<text x="{x:.1f}" y="{top+ploth+25}" text-anchor="middle" font-family="sans-serif" font-size="14">{n} replicas</text>')
out += [f'<line x1="130" y1="420" x2="160" y2="420" stroke="{colors[0]}" stroke-width="3"/><text x="168" y="424" font-family="sans-serif" font-size="13">{groups[0][0]}</text>',f'<line x1="430" y1="420" x2="460" y2="420" stroke="{colors[1]}" stroke-width="3"/><text x="468" y="424" font-family="sans-serif" font-size="13">{groups[1][0]}</text>','</svg>']
path.write_text("\n".join(out)+"\n")
if __name__=="__main__": main()
+188
View File
@@ -0,0 +1,188 @@
#!/usr/bin/env python3
"""Rate-controlled synthetic write test to locate the throughput knee."""
from concurrent.futures import ThreadPoolExecutor, as_completed
import json
import math
import random
import statistics
import subprocess
import threading
import time
from pathlib import Path
import sys
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
sys.path.insert(0, str(ROOT / "scripts"))
from client import Client
import track1_experiment as bench
OUT = ROOT / "evidence"
DURATION = 15
RATES = (250, 500, 750, 1000)
REPEATS = 3
def percentile(values, p):
values = sorted(values)
if not values:
return None
return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 3)
def run_sample(actor, chat, rate, repeat, reset=True):
if reset:
bench.reset_synthetic_messages()
latencies = []
failures = []
lock = threading.Lock()
local = threading.local()
start = time.monotonic()
deadline = start + DURATION
def request(index, due):
client = getattr(local, "client", None)
if client is None:
client = Client("http://127.0.0.1:8080", timeout=8)
local.client = client
sent = time.monotonic()
try:
client.message(actor, chat, f"offered-{rate}-{repeat}-{index}", key=f"offered-{rate}-{repeat}-{index}")
result = (time.monotonic() - sent, None)
except Exception as exc:
result = (None, f"{type(exc).__name__}: {str(exc)[:120]}")
with lock:
if result[0] is not None:
latencies.append(result[0])
else:
failures.append(result[1])
submitted = 0
offered_slots = 0
skipped_overload = 0
due = start
next_prune_at = start + .05
max_inflight = 8192
with ThreadPoolExecutor(max_workers=512) as pool:
futures = []
while due < deadline:
offered_slots += 1
if time.monotonic() >= next_prune_at:
futures = [future for future in futures if not future.done()]
next_prune_at = time.monotonic() + .05
if len(futures) < max_inflight:
futures.append(pool.submit(request, submitted, due))
submitted += 1
else:
skipped_overload += 1
due += 1 / rate
remaining = due - time.monotonic()
if remaining > 0:
time.sleep(remaining)
for future in as_completed(futures):
future.result()
elapsed = time.monotonic() - start
counts = {}
for failure in failures:
prefix = failure.split(":", 1)[0]
counts[prefix] = counts.get(prefix, 0) + 1
row_count_error = None
try:
after = bench.row_count()
except subprocess.CalledProcessError as exc:
after = None
row_count_error = (exc.stderr or "PostgreSQL row-count query failed").strip()[-300:]
return {
"variant": f"offered-{rate}", "sample": repeat, "offered_rate_rps": rate,
"duration_seconds": DURATION, "concurrency_workers": 512,
"max_inflight_requests": max_inflight, "offered_slots": offered_slots,
"scheduled_requests": submitted, "successful_requests": len(latencies),
"failed_requests": len(failures), "failure_types": counts,
"skipped_overload": skipped_overload,
"offered_rps_actual": round(offered_slots / DURATION, 3),
"submitted_rps_actual": round(submitted / DURATION, 3),
"completed_success_rps_total_elapsed": round(len(latencies) / elapsed, 3),
"success_ratio": round(len(latencies) / submitted, 6) if submitted else None,
"effective_success_ratio_offered": round(len(latencies) / offered_slots, 6) if offered_slots else None,
"latency_successful_ms": {f"p{p}": percentile(latencies, p/100) for p in (50,95,99)},
"drain_seconds_after_offer_window": round(max(0, elapsed - DURATION), 3),
"message_rows_after": after,
"message_rows_after_error": row_count_error,
}
def main():
OUT.mkdir(exist_ok=True)
env = os.environ.copy()
env.update(API_CPUS="0.5", API_MEMORY="256m", DB_POOL_MAX="10", LB_CPUS="0.25")
bench.run(bench.COMPOSE + ["up", "-d", "--build", "--scale", "api=8", "--wait", "--wait-timeout", "180"], env)
bench.wait_ready()
time.sleep(5)
client = Client("http://127.0.0.1:8080")
actor, _, _, chat = client.bootstrap()
rng = random.Random(20261106)
samples = []
for repeat in range(1, REPEATS + 1):
rates = list(RATES)
rng.shuffle(rates)
for rate in rates:
row = run_sample(actor, chat, rate, repeat)
samples.append(row)
print(json.dumps(row, ensure_ascii=False), flush=True)
summary = []
for rate in RATES:
group = [row for row in samples if row["offered_rate_rps"] == rate]
summary.append({
"offered_rate_rps": rate,
"completed_success_rps_total_elapsed_median": round(statistics.median(row["completed_success_rps_total_elapsed"] for row in group), 3),
"completed_success_rps_total_elapsed_min": min(row["completed_success_rps_total_elapsed"] for row in group),
"completed_success_rps_total_elapsed_max": max(row["completed_success_rps_total_elapsed"] for row in group),
"success_ratio_median": round(statistics.median(row["success_ratio"] for row in group), 6),
"p95_ms_median": round(statistics.median(row["latency_successful_ms"]["p95"] for row in group), 3),
"effective_success_ratio_offered_median": round(statistics.median(row["effective_success_ratio_offered"] for row in group), 6),
"skipped_overload_median": int(statistics.median(row["skipped_overload"] for row in group)),
})
result = {
"method": "Rate-controlled POST sweep; 15s per sample; offered rate 250/500/750/1000 RPS; three randomized repeats; 8 API replicas at 0.5 CPU/256 MiB, DB_POOL_MAX=10, PostgreSQL 2 CPU/1 GiB, HAProxy 0.25 CPU; max in-flight 8192; per-thread HTTP client reuse; synthetic message/idempotency tables reset per sample; no Docker stats polling during load.",
"summary": summary, "samples": samples,
}
(OUT / "offered-load-results.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
make_plot(summary, OUT / "offered-load.svg")
print(json.dumps(summary, ensure_ascii=False, indent=2))
def make_plot(rows, path):
w, h = 900, 500
left, right, top, bottom = 90, 90, 60, 85
pw, ph = w-left-right, h-top-bottom
max_rps, max_p95 = 1000, 2000
rps = [r["completed_success_rps_total_elapsed_median"] for r in rows]
p95 = [r["p95_ms_median"] for r in rows]
xs = [left + i*pw/(len(rows)-1) for i in range(len(rows))]
yr = [top + ph*(1-v/max_rps) for v in rps]
yp = [top + ph*(1-v/max_p95) for v in p95]
out = [f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" viewBox="0 0 {w} {h}">',
'<rect width="100%" height="100%" fill="#fff"/>',
'<text x="450" y="30" text-anchor="middle" font-family="sans-serif" font-size="20">Насыщение при росте предложенной нагрузки</text>']
for tick in (0,250,500,750,1000):
y=top+ph*(1-tick/max_rps)
out += [f'<line x1="{left}" y1="{y:.1f}" x2="{left+pw}" y2="{y:.1f}" stroke="#ddd"/>',
f'<text x="{left-12}" y="{y+5:.1f}" text-anchor="end" font-family="sans-serif" font-size="12">{tick}</text>']
for x, row in zip(xs, rows):
out += [f'<text x="{x:.1f}" y="{top+ph+24}" text-anchor="middle" font-family="sans-serif" font-size="12">{row["offered_rate_rps"]}</text>']
out += [f'<polyline points="{" ".join(f"{x:.1f},{y:.1f}" for x,y in zip(xs,yr))}" fill="none" stroke="#1769aa" stroke-width="3"/>',
f'<polyline points="{" ".join(f"{x:.1f},{y:.1f}" for x,y in zip(xs,yp))}" fill="none" stroke="#d1495b" stroke-width="3"/>']
for x,y,value in zip(xs,yr,rps):
out.append(f'<circle cx="{x:.1f}" cy="{y:.1f}" r="5" fill="#1769aa"/><text x="{x:.1f}" y="{y-12:.1f}" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#1769aa">{value:.0f}</text>')
for x,y,value in zip(xs,yp,p95):
out.append(f'<circle cx="{x:.1f}" cy="{y:.1f}" r="5" fill="#d1495b"/><text x="{x:.1f}" y="{y+19:.1f}" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#d1495b">{value:.0f}ms</text>')
out += [f'<text x="{left+pw/2:.1f}" y="{h-36}" text-anchor="middle" font-family="sans-serif" font-size="13">Предложенная нагрузка, RPS</text>',
f'<text x="22" y="{top+ph/2:.1f}" text-anchor="middle" transform="rotate(-90 22 {top+ph/2:.1f})" font-family="sans-serif" font-size="13">Успешные RPS за offer+drain</text>',
f'<text x="{w-22}" y="{top+ph/2:.1f}" text-anchor="middle" transform="rotate(90 {w-22} {top+ph/2:.1f})" font-family="sans-serif" font-size="13">p95, ms</text>',
f'<line x1="{left}" y1="{h-16}" x2="{left+25}" y2="{h-16}" stroke="#1769aa" stroke-width="3"/><text x="{left+33}" y="{h-12}" font-family="sans-serif" font-size="12">Успешные RPS</text>',
f'<line x1="{left+160}" y1="{h-16}" x2="{left+185}" y2="{h-16}" stroke="#d1495b" stroke-width="3"/><text x="{left+193}" y="{h-12}" font-family="sans-serif" font-size="12">p95 успешных запросов</text>',
'</svg>']
path.write_text('\n'.join(out)+'\n')
if __name__ == "__main__":
main()