Измерить отказ каждой реплики и цену добавления процессов.

Один и тот же профиль нагрузки сравнивает 1, 2 и 4 реплики при фиксированном и растущем бюджете CPU.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-11-07 12:46:37 +03:00
co-authored by Cursor
parent cc7147f458
commit 9373c08d74
23 changed files with 1365 additions and 0 deletions
+229
View File
@@ -0,0 +1,229 @@
#!/usr/bin/env python3
"""Grade 4: parallel idempotency, stop each API replica, rolling replacement under load."""
import json
import os
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
OUT = ROOT / "evidence" / "grade4"
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
def compose(*args, check=True):
cmd = ["docker", "compose"]
if (ROOT / ".env").is_file():
cmd.extend(["--env-file", str(ROOT / ".env")])
cmd.extend(args)
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
if check and proc.returncode != 0:
raise SystemExit("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or proc.stdout)[-800:]))
return proc
def bash_argv(script: Path):
if os.name != "nt":
return [str(script)]
git_bash = Path(r"C:\Program Files\Git\bin\bash.exe")
if git_bash.is_file():
return [str(git_bash), str(script)]
return ["bash", str(script)]
def hook_cmd():
return bash_argv(ROOT / "scripts" / "stop-one.sh")
def wait_ready(seconds=60):
deadline = time.monotonic() + seconds
last = None
while time.monotonic() < deadline:
try:
Client(BASE, timeout=3).call("GET", "/health/ready", schema="Health")
return
except Failure as exc:
last = exc
time.sleep(0.4)
raise SystemExit("API did not become ready: %s" % last)
def instance_ids(actor, samples=24):
found = []
client = Client(BASE, timeout=3)
for _ in range(samples):
found.append(client.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"]["X-Instance-Id"])
return sorted(set(found))
def parallel_idempotency():
client = Client(BASE, timeout=20)
alice, _bob, _eve, chat = client.bootstrap()
key = uuid.uuid4().hex
text = "parallel-" + key
ids = []
errors = []
barrier = threading.Barrier(16)
def one():
local = Client(BASE, timeout=20)
barrier.wait()
try:
ids.append(local.message(alice, chat, text, key=key)["json"]["id"])
except Exception as exc:
errors.append(type(exc).__name__)
threads = [threading.Thread(target=one) for _ in range(16)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
unique = sorted(set(ids))
report = {
"workers": 16,
"successes": len(ids),
"errors": len(errors),
"distinct_message_ids": len(unique),
}
if len(unique) != 1 or errors:
raise SystemExit("parallel idempotency failed: %s" % report)
page = client.call(
"GET",
"/api/v1/chats/%s/messages?limit=100" % chat["id"],
actor=alice,
schema="MessagePage",
)["json"]
copies = [item for item in page["items"] if item["text"] == text]
report["history_copies"] = len(copies)
if len(copies) != 1:
raise SystemExit("history has %s copies" % len(copies))
return report
def failover_one(actor, chat, instance_id):
key = "failover-" + uuid.uuid4().hex
text = "during-" + key
successes = []
failures = 0
started_box = {}
def traffic():
nonlocal failures
local = Client(BASE, timeout=2)
while "t0" not in started_box:
time.sleep(0.01)
deadline = started_box["t0"] + 12
recovered = False
while time.monotonic() < deadline:
try:
if not recovered:
response = local.message(actor, chat, text, key=key)
recovered = True
else:
response = local.message(actor, chat, "after-" + uuid.uuid4().hex)
successes.append((time.monotonic(), response["headers"]["X-Instance-Id"], response["json"]["id"]))
except Failure:
failures += 1
time.sleep(0.05)
worker = threading.Thread(target=traffic)
worker.start()
time.sleep(0.2)
started = time.monotonic()
started_box["t0"] = started
env = os.environ.copy()
env["TARGET_INSTANCE_ID"] = instance_id
proc = subprocess.run(hook_cmd(), cwd=ROOT, env=env, capture_output=True, text=True)
hook_s = time.monotonic() - started
worker.join()
after = [item for item in successes if item[0] >= started]
recovery = None
if after:
recovery = round(after[0][0] - started, 3)
tail = [item[1] for item in after[-5:]]
return {
"target": instance_id,
"hook_exit": proc.returncode,
"hook_seconds": round(hook_s, 3),
"hook_stderr": (proc.stderr or "")[-300:],
"failed_attempts": failures,
"successes_after_hook_start": len(after),
"first_recovery_seconds": recovery,
"tail_instance_ids": tail,
"target_still_in_tail": instance_id in tail,
}
def restore_and_wait(actor):
compose("start", "api")
deadline = time.monotonic() + 40
while time.monotonic() < deadline:
try:
if len(instance_ids(actor, samples=16)) >= 2:
return
except Failure:
pass
time.sleep(0.5)
raise SystemExit("second replica did not return")
def rolling_under_load():
out = OUT / "rolling-load.json"
load = subprocess.Popen(
[sys.executable, "-X", "utf8", str(ROOT / "tests" / "load.py"),
"--base-url", BASE, "--duration", "50", "--concurrency", "4", "--output", str(out)],
cwd=ROOT,
)
time.sleep(3)
cmd = bash_argv(ROOT / "scripts" / "rolling-update.sh")
started = time.monotonic()
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
elapsed = round(time.monotonic() - started, 3)
load.wait(timeout=90)
metrics = json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}
if proc.returncode != 0:
raise SystemExit("rolling update failed: %s" % (proc.stderr or proc.stdout)[-800:])
return {
"rolling_seconds": elapsed,
"rolling_stdout": (proc.stdout or "")[-400:],
"load": metrics,
}
def main():
OUT.mkdir(parents=True, exist_ok=True)
wait_ready()
client = Client(BASE, timeout=10)
alice, _bob, _eve, chat = client.bootstrap()
before = instance_ids(alice)
if len(before) < 2:
raise SystemExit("need 2 instance ids before grade-4 runs, saw %s" % before)
idem = parallel_idempotency()
failovers = []
for instance_id in before:
item = failover_one(alice, chat, instance_id)
failovers.append(item)
recovery = item["first_recovery_seconds"]
if item["hook_exit"] != 0 or item["target_still_in_tail"] or recovery is None or recovery > 10 or item["successes_after_hook_start"] < 5:
raise SystemExit("failover criteria failed: %s" % item)
restore_and_wait(alice)
rolling = rolling_under_load()
report = {
"instances_before": before,
"parallel_idempotency": idem,
"failovers": failovers,
"rolling": rolling,
}
(OUT / "summary.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+217
View File
@@ -0,0 +1,217 @@
#!/usr/bin/env python3
"""Track 1: fixed API budget, then more replicas with more CPU, then one DB change.
The load profile is the public closed-loop POST generator. Same duration and
concurrency on every point. Raw JSON is written under evidence/track1/.
"""
import json
import os
import subprocess
import sys
import threading
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "evidence" / "track1"
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
DURATION = float(os.environ.get("TRACK_DURATION", "15"))
WARMUP = float(os.environ.get("TRACK_WARMUP", "5"))
CONCURRENCY = int(os.environ.get("TRACK_CONCURRENCY", "8"))
def compose(*args):
cmd = ["docker", "compose"]
if (ROOT / ".env").is_file():
cmd.extend(["--env-file", str(ROOT / ".env")])
cmd.extend(args)
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, env=os.environ.copy())
if proc.returncode != 0:
raise SystemExit((proc.stderr or proc.stdout)[-1000:])
return proc
def wait_ready(seconds=90):
import urllib.request
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(BASE + "/health/ready", timeout=2) as response:
if response.status == 200:
return
except Exception:
time.sleep(0.5)
raise SystemExit("ready timeout")
def apply_profile(replicas, api_cpus, api_mem, postgres_cpus, pool):
os.environ["API_CPUS"] = api_cpus
os.environ["API_MEM"] = api_mem
os.environ["POSTGRES_CPUS"] = postgres_cpus
os.environ["DB_POOL_MAX"] = str(pool)
compose("up", "-d", "--no-deps", "--force-recreate", "postgres")
deadline = time.monotonic() + 40
while time.monotonic() < deadline:
probe = subprocess.run(
["docker", "compose", "--env-file", str(ROOT / ".env"), "exec", "-T", "postgres", "pg_isready", "-U", "messenger", "-d", "messenger"],
cwd=ROOT,
capture_output=True,
text=True,
)
if probe.returncode == 0:
break
time.sleep(0.5)
else:
raise SystemExit("postgres did not become ready")
compose(
"up",
"-d",
"--no-deps",
"--force-recreate",
"--scale",
"api=%s" % replicas,
"api",
)
wait_ready()
running = [line for line in compose("ps", "-q", "api").stdout.splitlines() if line.strip()]
if len(running) != replicas:
raise SystemExit("expected %s api containers, saw %s" % (replicas, len(running)))
def run_load(tag, warmup=False):
args = [
sys.executable,
"-X",
"utf8",
str(ROOT / "tests" / "load.py"),
"--base-url",
BASE,
"--duration",
str(WARMUP if warmup else DURATION),
"--concurrency",
str(CONCURRENCY),
]
dest = None
if not warmup:
dest = OUT / ("%s.json" % tag)
args.extend(["--output", str(dest)])
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True)
if proc.returncode != 0:
raise SystemExit("load %s failed: %s" % (tag, (proc.stderr or proc.stdout)[-800:]))
if dest is None:
return None
return json.loads(dest.read_text(encoding="utf-8"))
def sample_resources():
stats = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"],
capture_output=True,
text=True,
)
activity = compose(
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
"messenger",
"-tAc",
"SELECT count(*) FROM pg_stat_activity WHERE datname='messenger'",
)
return {
"docker_stats": [line for line in stats.stdout.splitlines() if line.strip()],
"postgres_sessions": activity.stdout.strip(),
}
def measure(tag, replicas, api_cpus, api_mem, postgres_cpus, pool):
print("profile %s" % tag, flush=True)
apply_profile(replicas, api_cpus, api_mem, postgres_cpus, pool)
run_load(tag, warmup=True)
box = {}
def snapshot():
time.sleep(DURATION / 2)
box["resources"] = sample_resources()
thread = threading.Thread(target=snapshot)
thread.start()
metrics = run_load(tag)
thread.join()
point = {
"tag": tag,
"replicas": replicas,
"api_cpus_each": api_cpus,
"api_mem_each": api_mem,
"api_cpu_total": round(replicas * float(api_cpus), 3),
"postgres_cpus": postgres_cpus,
"db_pool_max_each": pool,
"db_pool_max_total": replicas * pool,
"load": metrics,
"resources": box.get("resources"),
}
(OUT / ("%s.point.json" % tag)).write_text(json.dumps(point, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"tag": tag, "rps": metrics["successful_rps"], "errors": metrics["failed_requests"], "p95": metrics["successful_latency_ms"]["p95"]}), flush=True)
return point
def svg(points, path):
width, height = 720, 320
labels = [p["tag"] for p in points]
values = [p["load"]["successful_rps"] for p in points]
peak = max(values) or 1
left, top, plot_w, plot_h = 48, 20, width - 70, height - 70
bar_w = plot_w / len(points) * 0.62
gap = plot_w / len(points)
parts = [
'<?xml version="1.0" encoding="UTF-8"?>',
'<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d">' % (width, height),
'<rect width="100%" height="100%" fill="white"/>',
'<text x="48" y="16" font-family="sans-serif" font-size="14">Successful RPS, closed-loop POST, concurrency %d, %ss</text>'
% (CONCURRENCY, int(DURATION)),
]
for index, (label, value) in enumerate(zip(labels, values)):
h = plot_h * value / peak
x = left + index * gap + (gap - bar_w) / 2
y = top + 12 + (plot_h - h)
parts.append('<rect x="%.1f" y="%.1f" width="%.1f" height="%.1f" fill="#2f6f4e"/>' % (x, y, bar_w, h))
parts.append(
'<text x="%.1f" y="%.1f" font-family="sans-serif" font-size="12" text-anchor="middle">%.1f</text>'
% (x + bar_w / 2, y - 4, value)
)
parts.append(
'<text x="%.1f" y="%d" font-family="sans-serif" font-size="11" text-anchor="middle">%s</text>'
% (x + bar_w / 2, height - 18, label)
)
parts.append("</svg>")
path.write_text("\n".join(parts) + "\n", encoding="utf-8")
def main():
OUT.mkdir(parents=True, exist_ok=True)
points = []
# Series A: total API budget fixed at 1.0 CPU and 512 MiB. Pool total stays 8.
points.append(measure("A1", 1, "1.0", "512m", "1.0", 8))
points.append(measure("A2", 2, "0.50", "256m", "1.0", 4))
points.append(measure("A4", 4, "0.25", "128m", "1.0", 2))
# Series B: each new replica brings the same 0.50 CPU / 256 MiB. Postgres stays at 1 CPU.
points.append(measure("B1", 1, "0.50", "256m", "1.0", 4))
points.append(measure("B2", 2, "0.50", "256m", "1.0", 4))
points.append(measure("B4", 4, "0.50", "256m", "1.0", 4))
# Candidate improvement: give the database the extra CPU the API side already has.
points.append(measure("C4-pg2", 4, "0.50", "256m", "2.0", 4))
# Candidate improvement: stop the 4 pools from opening 16 server connections.
points.append(measure("D4-pool2", 4, "0.50", "256m", "1.0", 2))
(OUT / "summary.json").write_text(json.dumps(points, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
svg(points, OUT / "rps.svg")
# Return the stand to the documented 2 x 0.50 CPU layout.
apply_profile(2, "0.50", "256m", "1.0", 4)
print("wrote %s" % (OUT / "summary.json"))
if __name__ == "__main__":
main()