#!/usr/bin/env python3 """Rate-controlled synthetic write test to locate the throughput knee.""" from concurrent.futures import ThreadPoolExecutor, as_completed import json import math import random import statistics import subprocess import threading import time from pathlib import Path import sys ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) sys.path.insert(0, str(ROOT / "scripts")) from client import Client import track1_experiment as bench OUT = ROOT / "evidence" DURATION = 15 RATES = (250, 500, 750, 1000) REPEATS = 3 def percentile(values, p): values = sorted(values) if not values: return None return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 3) def run_sample(actor, chat, rate, repeat, reset=True): if reset: bench.reset_synthetic_messages() latencies = [] failures = [] lock = threading.Lock() local = threading.local() start = time.monotonic() deadline = start + DURATION def request(index, due): client = getattr(local, "client", None) if client is None: client = Client("http://127.0.0.1:8080", timeout=8) local.client = client sent = time.monotonic() try: client.message(actor, chat, f"offered-{rate}-{repeat}-{index}", key=f"offered-{rate}-{repeat}-{index}") result = (time.monotonic() - sent, None) except Exception as exc: result = (None, f"{type(exc).__name__}: {str(exc)[:120]}") with lock: if result[0] is not None: latencies.append(result[0]) else: failures.append(result[1]) submitted = 0 offered_slots = 0 skipped_overload = 0 due = start next_prune_at = start + .05 max_inflight = 8192 with ThreadPoolExecutor(max_workers=512) as pool: futures = [] while due < deadline: offered_slots += 1 if time.monotonic() >= next_prune_at: futures = [future for future in futures if not future.done()] next_prune_at = time.monotonic() + .05 if len(futures) < max_inflight: futures.append(pool.submit(request, submitted, due)) submitted += 1 else: skipped_overload += 1 due += 1 / rate remaining = due - time.monotonic() if remaining > 0: time.sleep(remaining) for future in as_completed(futures): future.result() elapsed = time.monotonic() - start counts = {} for failure in failures: prefix = failure.split(":", 1)[0] counts[prefix] = counts.get(prefix, 0) + 1 row_count_error = None try: after = bench.row_count() except subprocess.CalledProcessError as exc: after = None row_count_error = (exc.stderr or "PostgreSQL row-count query failed").strip()[-300:] return { "variant": f"offered-{rate}", "sample": repeat, "offered_rate_rps": rate, "duration_seconds": DURATION, "concurrency_workers": 512, "max_inflight_requests": max_inflight, "offered_slots": offered_slots, "scheduled_requests": submitted, "successful_requests": len(latencies), "failed_requests": len(failures), "failure_types": counts, "skipped_overload": skipped_overload, "offered_rps_actual": round(offered_slots / DURATION, 3), "submitted_rps_actual": round(submitted / DURATION, 3), "completed_success_rps_total_elapsed": round(len(latencies) / elapsed, 3), "success_ratio": round(len(latencies) / submitted, 6) if submitted else None, "effective_success_ratio_offered": round(len(latencies) / offered_slots, 6) if offered_slots else None, "latency_successful_ms": {f"p{p}": percentile(latencies, p/100) for p in (50,95,99)}, "drain_seconds_after_offer_window": round(max(0, elapsed - DURATION), 3), "message_rows_after": after, "message_rows_after_error": row_count_error, } def main(): OUT.mkdir(exist_ok=True) env = os.environ.copy() env.update(API_CPUS="0.5", API_MEMORY="256m", DB_POOL_MAX="10", LB_CPUS="0.25") bench.run(bench.COMPOSE + ["up", "-d", "--build", "--scale", "api=8", "--wait", "--wait-timeout", "180"], env) bench.wait_ready() time.sleep(5) client = Client("http://127.0.0.1:8080") actor, _, _, chat = client.bootstrap() rng = random.Random(20261106) samples = [] for repeat in range(1, REPEATS + 1): rates = list(RATES) rng.shuffle(rates) for rate in rates: row = run_sample(actor, chat, rate, repeat) samples.append(row) print(json.dumps(row, ensure_ascii=False), flush=True) summary = [] for rate in RATES: group = [row for row in samples if row["offered_rate_rps"] == rate] summary.append({ "offered_rate_rps": rate, "completed_success_rps_total_elapsed_median": round(statistics.median(row["completed_success_rps_total_elapsed"] for row in group), 3), "completed_success_rps_total_elapsed_min": min(row["completed_success_rps_total_elapsed"] for row in group), "completed_success_rps_total_elapsed_max": max(row["completed_success_rps_total_elapsed"] for row in group), "success_ratio_median": round(statistics.median(row["success_ratio"] for row in group), 6), "p95_ms_median": round(statistics.median(row["latency_successful_ms"]["p95"] for row in group), 3), "effective_success_ratio_offered_median": round(statistics.median(row["effective_success_ratio_offered"] for row in group), 6), "skipped_overload_median": int(statistics.median(row["skipped_overload"] for row in group)), }) result = { "method": "Rate-controlled POST sweep; 15s per sample; offered rate 250/500/750/1000 RPS; three randomized repeats; 8 API replicas at 0.5 CPU/256 MiB, DB_POOL_MAX=10, PostgreSQL 2 CPU/1 GiB, HAProxy 0.25 CPU; max in-flight 8192; per-thread HTTP client reuse; synthetic message/idempotency tables reset per sample; no Docker stats polling during load.", "summary": summary, "samples": samples, } (OUT / "offered-load-results.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") make_plot(summary, OUT / "offered-load.svg") print(json.dumps(summary, ensure_ascii=False, indent=2)) def make_plot(rows, path): w, h = 900, 500 left, right, top, bottom = 90, 90, 60, 85 pw, ph = w-left-right, h-top-bottom max_rps, max_p95 = 1000, 2000 rps = [r["completed_success_rps_total_elapsed_median"] for r in rows] p95 = [r["p95_ms_median"] for r in rows] xs = [left + i*pw/(len(rows)-1) for i in range(len(rows))] yr = [top + ph*(1-v/max_rps) for v in rps] yp = [top + ph*(1-v/max_p95) for v in p95] out = [f'', '', 'Насыщение при росте предложенной нагрузки'] for tick in (0,250,500,750,1000): y=top+ph*(1-tick/max_rps) out += [f'', f'{tick}'] for x, row in zip(xs, rows): out += [f'{row["offered_rate_rps"]}'] out += [f'', f''] for x,y,value in zip(xs,yr,rps): out.append(f'{value:.0f}') for x,y,value in zip(xs,yp,p95): out.append(f'{value:.0f}ms') out += [f'Предложенная нагрузка, RPS', f'Успешные RPS за offer+drain', f'p95, ms', f'Успешные RPS', f'p95 успешных запросов', ''] path.write_text('\n'.join(out)+'\n') if __name__ == "__main__": main()