forked from mtusi2026/mtusi2026lab3
189 lines
9.4 KiB
Python
189 lines
9.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Rate-controlled synthetic write test to locate the throughput knee."""
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import json
|
|
import math
|
|
import random
|
|
import statistics
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
from client import Client
|
|
import track1_experiment as bench
|
|
OUT = ROOT / "evidence"
|
|
DURATION = 15
|
|
RATES = (250, 500, 750, 1000)
|
|
REPEATS = 3
|
|
|
|
|
|
def percentile(values, p):
|
|
values = sorted(values)
|
|
if not values:
|
|
return None
|
|
return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 3)
|
|
|
|
|
|
def run_sample(actor, chat, rate, repeat, reset=True):
|
|
if reset:
|
|
bench.reset_synthetic_messages()
|
|
latencies = []
|
|
failures = []
|
|
lock = threading.Lock()
|
|
local = threading.local()
|
|
start = time.monotonic()
|
|
deadline = start + DURATION
|
|
|
|
def request(index, due):
|
|
client = getattr(local, "client", None)
|
|
if client is None:
|
|
client = Client("http://127.0.0.1:8080", timeout=8)
|
|
local.client = client
|
|
sent = time.monotonic()
|
|
try:
|
|
client.message(actor, chat, f"offered-{rate}-{repeat}-{index}", key=f"offered-{rate}-{repeat}-{index}")
|
|
result = (time.monotonic() - sent, None)
|
|
except Exception as exc:
|
|
result = (None, f"{type(exc).__name__}: {str(exc)[:120]}")
|
|
with lock:
|
|
if result[0] is not None:
|
|
latencies.append(result[0])
|
|
else:
|
|
failures.append(result[1])
|
|
|
|
submitted = 0
|
|
offered_slots = 0
|
|
skipped_overload = 0
|
|
due = start
|
|
next_prune_at = start + .05
|
|
max_inflight = 8192
|
|
with ThreadPoolExecutor(max_workers=512) as pool:
|
|
futures = []
|
|
while due < deadline:
|
|
offered_slots += 1
|
|
if time.monotonic() >= next_prune_at:
|
|
futures = [future for future in futures if not future.done()]
|
|
next_prune_at = time.monotonic() + .05
|
|
if len(futures) < max_inflight:
|
|
futures.append(pool.submit(request, submitted, due))
|
|
submitted += 1
|
|
else:
|
|
skipped_overload += 1
|
|
due += 1 / rate
|
|
remaining = due - time.monotonic()
|
|
if remaining > 0:
|
|
time.sleep(remaining)
|
|
for future in as_completed(futures):
|
|
future.result()
|
|
elapsed = time.monotonic() - start
|
|
counts = {}
|
|
for failure in failures:
|
|
prefix = failure.split(":", 1)[0]
|
|
counts[prefix] = counts.get(prefix, 0) + 1
|
|
row_count_error = None
|
|
try:
|
|
after = bench.row_count()
|
|
except subprocess.CalledProcessError as exc:
|
|
after = None
|
|
row_count_error = (exc.stderr or "PostgreSQL row-count query failed").strip()[-300:]
|
|
return {
|
|
"variant": f"offered-{rate}", "sample": repeat, "offered_rate_rps": rate,
|
|
"duration_seconds": DURATION, "concurrency_workers": 512,
|
|
"max_inflight_requests": max_inflight, "offered_slots": offered_slots,
|
|
"scheduled_requests": submitted, "successful_requests": len(latencies),
|
|
"failed_requests": len(failures), "failure_types": counts,
|
|
"skipped_overload": skipped_overload,
|
|
"offered_rps_actual": round(offered_slots / DURATION, 3),
|
|
"submitted_rps_actual": round(submitted / DURATION, 3),
|
|
"completed_success_rps_total_elapsed": round(len(latencies) / elapsed, 3),
|
|
"success_ratio": round(len(latencies) / submitted, 6) if submitted else None,
|
|
"effective_success_ratio_offered": round(len(latencies) / offered_slots, 6) if offered_slots else None,
|
|
"latency_successful_ms": {f"p{p}": percentile(latencies, p/100) for p in (50,95,99)},
|
|
"drain_seconds_after_offer_window": round(max(0, elapsed - DURATION), 3),
|
|
"message_rows_after": after,
|
|
"message_rows_after_error": row_count_error,
|
|
}
|
|
|
|
|
|
def main():
|
|
OUT.mkdir(exist_ok=True)
|
|
env = os.environ.copy()
|
|
env.update(API_CPUS="0.5", API_MEMORY="256m", DB_POOL_MAX="10", LB_CPUS="0.25")
|
|
bench.run(bench.COMPOSE + ["up", "-d", "--build", "--scale", "api=8", "--wait", "--wait-timeout", "180"], env)
|
|
bench.wait_ready()
|
|
time.sleep(5)
|
|
client = Client("http://127.0.0.1:8080")
|
|
actor, _, _, chat = client.bootstrap()
|
|
rng = random.Random(20261106)
|
|
samples = []
|
|
for repeat in range(1, REPEATS + 1):
|
|
rates = list(RATES)
|
|
rng.shuffle(rates)
|
|
for rate in rates:
|
|
row = run_sample(actor, chat, rate, repeat)
|
|
samples.append(row)
|
|
print(json.dumps(row, ensure_ascii=False), flush=True)
|
|
summary = []
|
|
for rate in RATES:
|
|
group = [row for row in samples if row["offered_rate_rps"] == rate]
|
|
summary.append({
|
|
"offered_rate_rps": rate,
|
|
"completed_success_rps_total_elapsed_median": round(statistics.median(row["completed_success_rps_total_elapsed"] for row in group), 3),
|
|
"completed_success_rps_total_elapsed_min": min(row["completed_success_rps_total_elapsed"] for row in group),
|
|
"completed_success_rps_total_elapsed_max": max(row["completed_success_rps_total_elapsed"] for row in group),
|
|
"success_ratio_median": round(statistics.median(row["success_ratio"] for row in group), 6),
|
|
"p95_ms_median": round(statistics.median(row["latency_successful_ms"]["p95"] for row in group), 3),
|
|
"effective_success_ratio_offered_median": round(statistics.median(row["effective_success_ratio_offered"] for row in group), 6),
|
|
"skipped_overload_median": int(statistics.median(row["skipped_overload"] for row in group)),
|
|
})
|
|
result = {
|
|
"method": "Rate-controlled POST sweep; 15s per sample; offered rate 250/500/750/1000 RPS; three randomized repeats; 8 API replicas at 0.5 CPU/256 MiB, DB_POOL_MAX=10, PostgreSQL 2 CPU/1 GiB, HAProxy 0.25 CPU; max in-flight 8192; per-thread HTTP client reuse; synthetic message/idempotency tables reset per sample; no Docker stats polling during load.",
|
|
"summary": summary, "samples": samples,
|
|
}
|
|
(OUT / "offered-load-results.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
|
make_plot(summary, OUT / "offered-load.svg")
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
|
|
|
|
def make_plot(rows, path):
|
|
w, h = 900, 500
|
|
left, right, top, bottom = 90, 90, 60, 85
|
|
pw, ph = w-left-right, h-top-bottom
|
|
max_rps, max_p95 = 1000, 2000
|
|
rps = [r["completed_success_rps_total_elapsed_median"] for r in rows]
|
|
p95 = [r["p95_ms_median"] for r in rows]
|
|
xs = [left + i*pw/(len(rows)-1) for i in range(len(rows))]
|
|
yr = [top + ph*(1-v/max_rps) for v in rps]
|
|
yp = [top + ph*(1-v/max_p95) for v in p95]
|
|
out = [f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" viewBox="0 0 {w} {h}">',
|
|
'<rect width="100%" height="100%" fill="#fff"/>',
|
|
'<text x="450" y="30" text-anchor="middle" font-family="sans-serif" font-size="20">Насыщение при росте предложенной нагрузки</text>']
|
|
for tick in (0,250,500,750,1000):
|
|
y=top+ph*(1-tick/max_rps)
|
|
out += [f'<line x1="{left}" y1="{y:.1f}" x2="{left+pw}" y2="{y:.1f}" stroke="#ddd"/>',
|
|
f'<text x="{left-12}" y="{y+5:.1f}" text-anchor="end" font-family="sans-serif" font-size="12">{tick}</text>']
|
|
for x, row in zip(xs, rows):
|
|
out += [f'<text x="{x:.1f}" y="{top+ph+24}" text-anchor="middle" font-family="sans-serif" font-size="12">{row["offered_rate_rps"]}</text>']
|
|
out += [f'<polyline points="{" ".join(f"{x:.1f},{y:.1f}" for x,y in zip(xs,yr))}" fill="none" stroke="#1769aa" stroke-width="3"/>',
|
|
f'<polyline points="{" ".join(f"{x:.1f},{y:.1f}" for x,y in zip(xs,yp))}" fill="none" stroke="#d1495b" stroke-width="3"/>']
|
|
for x,y,value in zip(xs,yr,rps):
|
|
out.append(f'<circle cx="{x:.1f}" cy="{y:.1f}" r="5" fill="#1769aa"/><text x="{x:.1f}" y="{y-12:.1f}" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#1769aa">{value:.0f}</text>')
|
|
for x,y,value in zip(xs,yp,p95):
|
|
out.append(f'<circle cx="{x:.1f}" cy="{y:.1f}" r="5" fill="#d1495b"/><text x="{x:.1f}" y="{y+19:.1f}" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#d1495b">{value:.0f}ms</text>')
|
|
out += [f'<text x="{left+pw/2:.1f}" y="{h-36}" text-anchor="middle" font-family="sans-serif" font-size="13">Предложенная нагрузка, RPS</text>',
|
|
f'<text x="22" y="{top+ph/2:.1f}" text-anchor="middle" transform="rotate(-90 22 {top+ph/2:.1f})" font-family="sans-serif" font-size="13">Успешные RPS за offer+drain</text>',
|
|
f'<text x="{w-22}" y="{top+ph/2:.1f}" text-anchor="middle" transform="rotate(90 {w-22} {top+ph/2:.1f})" font-family="sans-serif" font-size="13">p95, ms</text>',
|
|
f'<line x1="{left}" y1="{h-16}" x2="{left+25}" y2="{h-16}" stroke="#1769aa" stroke-width="3"/><text x="{left+33}" y="{h-12}" font-family="sans-serif" font-size="12">Успешные RPS</text>',
|
|
f'<line x1="{left+160}" y1="{h-16}" x2="{left+185}" y2="{h-16}" stroke="#d1495b" stroke-width="3"/><text x="{left+193}" y="{h-12}" font-family="sans-serif" font-size="12">p95 успешных запросов</text>',
|
|
'</svg>']
|
|
path.write_text('\n'.join(out)+'\n')
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|