diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..cbc7894d --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +# Restart every stateful dependency and all API processes without deleting volumes. +docker compose --env-file .env restart api haproxy postgres valkey diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100644 index 00000000..214a8246 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,25 @@ +#!/bin/sh +set -eu +compose="docker compose --env-file .env" +# Add one ready replica before removing any old container. +$compose up -d --scale api=3 --wait --wait-timeout 120 api +python3 - <<'PY' +import time, urllib.request +end=time.monotonic()+60 +while time.monotonic()/dev/null + docker rm "$id" >/dev/null + docker compose --env-file .env up -d --scale api=3 --wait --wait-timeout 120 api + sleep 3 +done +docker compose --env-file .env up -d --scale api=2 --wait --wait-timeout 120 api diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..8c4acd14 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" +# X-Instance-Id is the container hostname (the Docker short ID). Resolve it to +# exactly one scaled Compose container and stop only that process. +for id in $(docker compose --env-file .env ps -q api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id") + if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then + docker stop --time 3 "$id" >/dev/null + exit 0 + fi +done +echo "No running api container matches TARGET_INSTANCE_ID" >&2 +exit 1 diff --git a/scripts/track1_experiment.py b/scripts/track1_experiment.py new file mode 100644 index 00000000..b00e8c4b --- /dev/null +++ b/scripts/track1_experiment.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Repeatable local scaling study: fixed aggregate budget, then increased budget.""" +import json, os, platform, random, subprocess, sys, time, urllib.request +from statistics import median +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +COMPOSE = ["docker", "compose", "--env-file", ".env"] +OUT = ROOT / "evidence" +LOAD = [sys.executable, "tests/load.py", "--base-url", "http://127.0.0.1:8080", "--duration", os.getenv("BENCH_SECONDS", "30"), "--concurrency", os.getenv("BENCH_CONCURRENCY", "16")] + +def run(args, env=None, check=True): + subprocess.run(args, cwd=ROOT, env=env, check=check) + +def docker_usage(): + ids=[] + for service in ("api","postgres","haproxy"): + found=subprocess.run(COMPOSE+["ps","-q",service],cwd=ROOT,text=True,capture_output=True,check=True).stdout.split() + ids.extend(found) + if not ids: return {} + raw=subprocess.run(["docker","stats","--no-stream","--format","{{json .}}",*ids],cwd=ROOT,text=True,capture_output=True,check=True).stdout + grouped={"api":{"cpu_percent":[],"memory_mib":[]},"postgres":{"cpu_percent":[],"memory_mib":[]},"haproxy":{"cpu_percent":[],"memory_mib":[]}} + for line in raw.splitlines(): + try: item=json.loads(line) + except ValueError: continue + name=item.get("Name","") + role="api" if "-api-" in name else "postgres" if "-postgres-" in name else "haproxy" if "-haproxy-" in name else None + if not role: continue + try: grouped[role]["cpu_percent"].append(float(item["CPUPerc"].rstrip("%"))) + except (KeyError,ValueError): pass + try: + size=item["MemUsage"].split("/")[0].strip() + if size.endswith("GiB"): value=float(size[:-3])*1024 + elif size.endswith("MiB"): value=float(size[:-3]) + elif size.endswith("KiB"): value=float(size[:-3])/1024 + else: value=float(size[:-1])/(1024*1024) + grouped[role]["memory_mib"].append(value) + except (KeyError,ValueError,IndexError): pass + return grouped + +def row_count(): + env={} + for line in (ROOT/".env").read_text().splitlines(): + if "=" in line: + k,v=line.split("=",1); env[k]=v + cmd=COMPOSE+["exec","-T","postgres","psql","-U",env.get("POSTGRES_USER","messenger"),"-d",env.get("POSTGRES_DB","messenger"),"-Atc","SELECT count(*) FROM messages"] + result=subprocess.run(cmd,cwd=ROOT,text=True,capture_output=True,check=True) + return int(result.stdout.strip()) + +def reset_synthetic_messages(): + """Use an identical empty message/idempotency dataset for every sample.""" + env={} + for line in (ROOT/".env").read_text().splitlines(): + if "=" in line: + k,v=line.split("=",1); env[k]=v + cmd=COMPOSE+["exec","-T","postgres","psql","-U",env.get("POSTGRES_USER","messenger"),"-d",env.get("POSTGRES_DB","messenger"),"-v","ON_ERROR_STOP=1","-c","TRUNCATE TABLE messages, idempotency_keys; ANALYZE messages; ANALYZE idempotency_keys;"] + subprocess.run(cmd,cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,check=True) + +def benchmark_load(dest, env): + reset_synthetic_messages() + before=row_count(); samples=[] + proc=subprocess.Popen(LOAD+["--output",str(dest)],cwd=ROOT,env=env,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,text=True) + while proc.poll() is None: + stats=docker_usage() + if stats: samples.append(stats) + time.sleep(1) + output=proc.communicate()[0] + print(output,end="") + if proc.returncode: raise subprocess.CalledProcessError(proc.returncode,LOAD) + after=row_count() + flat={} + for role in ("api","postgres","haproxy"): + cpus=[x for sample in samples for x in sample[role]["cpu_percent"]] + mems=[x for sample in samples for x in sample[role]["memory_mib"]] + flat[role]={"cpu_percent_mean":round(sum(cpus)/len(cpus),2) if cpus else None,"cpu_percent_peak":round(max(cpus),2) if cpus else None,"memory_mib_peak":round(max(mems),2) if mems else None} + return {"message_rows_before":before,"message_rows_after":after,"docker_stats_samples":len(samples),"container_resource_usage":flat} + +def wait_ready(timeout=90): + deadline=time.monotonic()+timeout + while time.monotonic()','','API scaling: successful requests per second'] + for tick in range(5): + y=top+ploth-tick*ploth/4; val=vmax*tick/4 + out += [f'',f'{val:.1f}'] + colors=["#2563eb","#f97316"] + for gi,(label,items) in enumerate(groups): + pts=[] + for i,r in enumerate(items): + x=left+plotw*({1:1,2:2,4:3,8:4}[r["replicas"]])/5; y=top+ploth-(r["successful_rps"]/vmax)*ploth + pts.append(f"{x:.1f},{y:.1f}"); out.append(f''); out.append(f'{r["successful_rps"]:.1f}') + out.append(f'') + positions={1:1,2:2,4:3,8:4} + for n,pos in positions.items(): + x=left+plotw*pos/5; out.append(f'{n} replicas') + out += [f'{groups[0][0]}',f'{groups[1][0]}',''] + path.write_text("\n".join(out)+"\n") + +if __name__=="__main__": main() diff --git a/scripts/track1_offered_load.py b/scripts/track1_offered_load.py new file mode 100644 index 00000000..e0e001a0 --- /dev/null +++ b/scripts/track1_offered_load.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Rate-controlled synthetic write test to locate the throughput knee.""" +from concurrent.futures import ThreadPoolExecutor, as_completed +import json +import math +import random +import statistics +import subprocess +import threading +import time +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +sys.path.insert(0, str(ROOT / "scripts")) +from client import Client +import track1_experiment as bench +OUT = ROOT / "evidence" +DURATION = 15 +RATES = (250, 500, 750, 1000) +REPEATS = 3 + + +def percentile(values, p): + values = sorted(values) + if not values: + return None + return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 3) + + +def run_sample(actor, chat, rate, repeat, reset=True): + if reset: + bench.reset_synthetic_messages() + latencies = [] + failures = [] + lock = threading.Lock() + local = threading.local() + start = time.monotonic() + deadline = start + DURATION + + def request(index, due): + client = getattr(local, "client", None) + if client is None: + client = Client("http://127.0.0.1:8080", timeout=8) + local.client = client + sent = time.monotonic() + try: + client.message(actor, chat, f"offered-{rate}-{repeat}-{index}", key=f"offered-{rate}-{repeat}-{index}") + result = (time.monotonic() - sent, None) + except Exception as exc: + result = (None, f"{type(exc).__name__}: {str(exc)[:120]}") + with lock: + if result[0] is not None: + latencies.append(result[0]) + else: + failures.append(result[1]) + + submitted = 0 + offered_slots = 0 + skipped_overload = 0 + due = start + next_prune_at = start + .05 + max_inflight = 8192 + with ThreadPoolExecutor(max_workers=512) as pool: + futures = [] + while due < deadline: + offered_slots += 1 + if time.monotonic() >= next_prune_at: + futures = [future for future in futures if not future.done()] + next_prune_at = time.monotonic() + .05 + if len(futures) < max_inflight: + futures.append(pool.submit(request, submitted, due)) + submitted += 1 + else: + skipped_overload += 1 + due += 1 / rate + remaining = due - time.monotonic() + if remaining > 0: + time.sleep(remaining) + for future in as_completed(futures): + future.result() + elapsed = time.monotonic() - start + counts = {} + for failure in failures: + prefix = failure.split(":", 1)[0] + counts[prefix] = counts.get(prefix, 0) + 1 + row_count_error = None + try: + after = bench.row_count() + except subprocess.CalledProcessError as exc: + after = None + row_count_error = (exc.stderr or "PostgreSQL row-count query failed").strip()[-300:] + return { + "variant": f"offered-{rate}", "sample": repeat, "offered_rate_rps": rate, + "duration_seconds": DURATION, "concurrency_workers": 512, + "max_inflight_requests": max_inflight, "offered_slots": offered_slots, + "scheduled_requests": submitted, "successful_requests": len(latencies), + "failed_requests": len(failures), "failure_types": counts, + "skipped_overload": skipped_overload, + "offered_rps_actual": round(offered_slots / DURATION, 3), + "submitted_rps_actual": round(submitted / DURATION, 3), + "completed_success_rps_total_elapsed": round(len(latencies) / elapsed, 3), + "success_ratio": round(len(latencies) / submitted, 6) if submitted else None, + "effective_success_ratio_offered": round(len(latencies) / offered_slots, 6) if offered_slots else None, + "latency_successful_ms": {f"p{p}": percentile(latencies, p/100) for p in (50,95,99)}, + "drain_seconds_after_offer_window": round(max(0, elapsed - DURATION), 3), + "message_rows_after": after, + "message_rows_after_error": row_count_error, + } + + +def main(): + OUT.mkdir(exist_ok=True) + env = os.environ.copy() + env.update(API_CPUS="0.5", API_MEMORY="256m", DB_POOL_MAX="10", LB_CPUS="0.25") + bench.run(bench.COMPOSE + ["up", "-d", "--build", "--scale", "api=8", "--wait", "--wait-timeout", "180"], env) + bench.wait_ready() + time.sleep(5) + client = Client("http://127.0.0.1:8080") + actor, _, _, chat = client.bootstrap() + rng = random.Random(20261106) + samples = [] + for repeat in range(1, REPEATS + 1): + rates = list(RATES) + rng.shuffle(rates) + for rate in rates: + row = run_sample(actor, chat, rate, repeat) + samples.append(row) + print(json.dumps(row, ensure_ascii=False), flush=True) + summary = [] + for rate in RATES: + group = [row for row in samples if row["offered_rate_rps"] == rate] + summary.append({ + "offered_rate_rps": rate, + "completed_success_rps_total_elapsed_median": round(statistics.median(row["completed_success_rps_total_elapsed"] for row in group), 3), + "completed_success_rps_total_elapsed_min": min(row["completed_success_rps_total_elapsed"] for row in group), + "completed_success_rps_total_elapsed_max": max(row["completed_success_rps_total_elapsed"] for row in group), + "success_ratio_median": round(statistics.median(row["success_ratio"] for row in group), 6), + "p95_ms_median": round(statistics.median(row["latency_successful_ms"]["p95"] for row in group), 3), + "effective_success_ratio_offered_median": round(statistics.median(row["effective_success_ratio_offered"] for row in group), 6), + "skipped_overload_median": int(statistics.median(row["skipped_overload"] for row in group)), + }) + result = { + "method": "Rate-controlled POST sweep; 15s per sample; offered rate 250/500/750/1000 RPS; three randomized repeats; 8 API replicas at 0.5 CPU/256 MiB, DB_POOL_MAX=10, PostgreSQL 2 CPU/1 GiB, HAProxy 0.25 CPU; max in-flight 8192; per-thread HTTP client reuse; synthetic message/idempotency tables reset per sample; no Docker stats polling during load.", + "summary": summary, "samples": samples, + } + (OUT / "offered-load-results.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + make_plot(summary, OUT / "offered-load.svg") + print(json.dumps(summary, ensure_ascii=False, indent=2)) + + +def make_plot(rows, path): + w, h = 900, 500 + left, right, top, bottom = 90, 90, 60, 85 + pw, ph = w-left-right, h-top-bottom + max_rps, max_p95 = 1000, 2000 + rps = [r["completed_success_rps_total_elapsed_median"] for r in rows] + p95 = [r["p95_ms_median"] for r in rows] + xs = [left + i*pw/(len(rows)-1) for i in range(len(rows))] + yr = [top + ph*(1-v/max_rps) for v in rps] + yp = [top + ph*(1-v/max_p95) for v in p95] + out = [f'', + '', + 'Насыщение при росте предложенной нагрузки'] + for tick in (0,250,500,750,1000): + y=top+ph*(1-tick/max_rps) + out += [f'', + f'{tick}'] + for x, row in zip(xs, rows): + out += [f'{row["offered_rate_rps"]}'] + out += [f'', + f''] + for x,y,value in zip(xs,yr,rps): + out.append(f'{value:.0f}') + for x,y,value in zip(xs,yp,p95): + out.append(f'{value:.0f}ms') + out += [f'Предложенная нагрузка, RPS', + f'Успешные RPS за offer+drain', + f'p95, ms', + f'Успешные RPS', + f'p95 успешных запросов', + ''] + path.write_text('\n'.join(out)+'\n') + + +if __name__ == "__main__": + main()