forked from mtusi2026/mtusi2026lab3
Один и тот же профиль нагрузки сравнивает 1, 2 и 4 реплики при фиксированном и растущем бюджете CPU. Co-authored-by: Cursor <cursoragent@cursor.com>
230 lines
7.3 KiB
Python
230 lines
7.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Grade 4: parallel idempotency, stop each API replica, rolling replacement under load."""
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
|
|
from client import Client, Failure # noqa: E402
|
|
|
|
OUT = ROOT / "evidence" / "grade4"
|
|
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
|
|
|
|
|
def compose(*args, check=True):
|
|
cmd = ["docker", "compose"]
|
|
if (ROOT / ".env").is_file():
|
|
cmd.extend(["--env-file", str(ROOT / ".env")])
|
|
cmd.extend(args)
|
|
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
|
|
if check and proc.returncode != 0:
|
|
raise SystemExit("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or proc.stdout)[-800:]))
|
|
return proc
|
|
|
|
|
|
def bash_argv(script: Path):
|
|
if os.name != "nt":
|
|
return [str(script)]
|
|
git_bash = Path(r"C:\Program Files\Git\bin\bash.exe")
|
|
if git_bash.is_file():
|
|
return [str(git_bash), str(script)]
|
|
return ["bash", str(script)]
|
|
|
|
|
|
def hook_cmd():
|
|
return bash_argv(ROOT / "scripts" / "stop-one.sh")
|
|
|
|
|
|
def wait_ready(seconds=60):
|
|
deadline = time.monotonic() + seconds
|
|
last = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
Client(BASE, timeout=3).call("GET", "/health/ready", schema="Health")
|
|
return
|
|
except Failure as exc:
|
|
last = exc
|
|
time.sleep(0.4)
|
|
raise SystemExit("API did not become ready: %s" % last)
|
|
|
|
|
|
def instance_ids(actor, samples=24):
|
|
found = []
|
|
client = Client(BASE, timeout=3)
|
|
for _ in range(samples):
|
|
found.append(client.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"]["X-Instance-Id"])
|
|
return sorted(set(found))
|
|
|
|
|
|
def parallel_idempotency():
|
|
client = Client(BASE, timeout=20)
|
|
alice, _bob, _eve, chat = client.bootstrap()
|
|
key = uuid.uuid4().hex
|
|
text = "parallel-" + key
|
|
ids = []
|
|
errors = []
|
|
barrier = threading.Barrier(16)
|
|
|
|
def one():
|
|
local = Client(BASE, timeout=20)
|
|
barrier.wait()
|
|
try:
|
|
ids.append(local.message(alice, chat, text, key=key)["json"]["id"])
|
|
except Exception as exc:
|
|
errors.append(type(exc).__name__)
|
|
|
|
threads = [threading.Thread(target=one) for _ in range(16)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join()
|
|
unique = sorted(set(ids))
|
|
report = {
|
|
"workers": 16,
|
|
"successes": len(ids),
|
|
"errors": len(errors),
|
|
"distinct_message_ids": len(unique),
|
|
}
|
|
if len(unique) != 1 or errors:
|
|
raise SystemExit("parallel idempotency failed: %s" % report)
|
|
page = client.call(
|
|
"GET",
|
|
"/api/v1/chats/%s/messages?limit=100" % chat["id"],
|
|
actor=alice,
|
|
schema="MessagePage",
|
|
)["json"]
|
|
copies = [item for item in page["items"] if item["text"] == text]
|
|
report["history_copies"] = len(copies)
|
|
if len(copies) != 1:
|
|
raise SystemExit("history has %s copies" % len(copies))
|
|
return report
|
|
|
|
|
|
def failover_one(actor, chat, instance_id):
|
|
key = "failover-" + uuid.uuid4().hex
|
|
text = "during-" + key
|
|
successes = []
|
|
failures = 0
|
|
started_box = {}
|
|
|
|
def traffic():
|
|
nonlocal failures
|
|
local = Client(BASE, timeout=2)
|
|
while "t0" not in started_box:
|
|
time.sleep(0.01)
|
|
deadline = started_box["t0"] + 12
|
|
recovered = False
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
if not recovered:
|
|
response = local.message(actor, chat, text, key=key)
|
|
recovered = True
|
|
else:
|
|
response = local.message(actor, chat, "after-" + uuid.uuid4().hex)
|
|
successes.append((time.monotonic(), response["headers"]["X-Instance-Id"], response["json"]["id"]))
|
|
except Failure:
|
|
failures += 1
|
|
time.sleep(0.05)
|
|
|
|
worker = threading.Thread(target=traffic)
|
|
worker.start()
|
|
time.sleep(0.2)
|
|
started = time.monotonic()
|
|
started_box["t0"] = started
|
|
env = os.environ.copy()
|
|
env["TARGET_INSTANCE_ID"] = instance_id
|
|
proc = subprocess.run(hook_cmd(), cwd=ROOT, env=env, capture_output=True, text=True)
|
|
hook_s = time.monotonic() - started
|
|
worker.join()
|
|
after = [item for item in successes if item[0] >= started]
|
|
recovery = None
|
|
if after:
|
|
recovery = round(after[0][0] - started, 3)
|
|
tail = [item[1] for item in after[-5:]]
|
|
return {
|
|
"target": instance_id,
|
|
"hook_exit": proc.returncode,
|
|
"hook_seconds": round(hook_s, 3),
|
|
"hook_stderr": (proc.stderr or "")[-300:],
|
|
"failed_attempts": failures,
|
|
"successes_after_hook_start": len(after),
|
|
"first_recovery_seconds": recovery,
|
|
"tail_instance_ids": tail,
|
|
"target_still_in_tail": instance_id in tail,
|
|
}
|
|
|
|
|
|
def restore_and_wait(actor):
|
|
compose("start", "api")
|
|
deadline = time.monotonic() + 40
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
if len(instance_ids(actor, samples=16)) >= 2:
|
|
return
|
|
except Failure:
|
|
pass
|
|
time.sleep(0.5)
|
|
raise SystemExit("second replica did not return")
|
|
|
|
|
|
def rolling_under_load():
|
|
out = OUT / "rolling-load.json"
|
|
load = subprocess.Popen(
|
|
[sys.executable, "-X", "utf8", str(ROOT / "tests" / "load.py"),
|
|
"--base-url", BASE, "--duration", "50", "--concurrency", "4", "--output", str(out)],
|
|
cwd=ROOT,
|
|
)
|
|
time.sleep(3)
|
|
cmd = bash_argv(ROOT / "scripts" / "rolling-update.sh")
|
|
started = time.monotonic()
|
|
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
|
|
elapsed = round(time.monotonic() - started, 3)
|
|
load.wait(timeout=90)
|
|
metrics = json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}
|
|
if proc.returncode != 0:
|
|
raise SystemExit("rolling update failed: %s" % (proc.stderr or proc.stdout)[-800:])
|
|
return {
|
|
"rolling_seconds": elapsed,
|
|
"rolling_stdout": (proc.stdout or "")[-400:],
|
|
"load": metrics,
|
|
}
|
|
|
|
|
|
def main():
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
wait_ready()
|
|
client = Client(BASE, timeout=10)
|
|
alice, _bob, _eve, chat = client.bootstrap()
|
|
before = instance_ids(alice)
|
|
if len(before) < 2:
|
|
raise SystemExit("need 2 instance ids before grade-4 runs, saw %s" % before)
|
|
idem = parallel_idempotency()
|
|
failovers = []
|
|
for instance_id in before:
|
|
item = failover_one(alice, chat, instance_id)
|
|
failovers.append(item)
|
|
recovery = item["first_recovery_seconds"]
|
|
if item["hook_exit"] != 0 or item["target_still_in_tail"] or recovery is None or recovery > 10 or item["successes_after_hook_start"] < 5:
|
|
raise SystemExit("failover criteria failed: %s" % item)
|
|
restore_and_wait(alice)
|
|
rolling = rolling_under_load()
|
|
report = {
|
|
"instances_before": before,
|
|
"parallel_idempotency": idem,
|
|
"failovers": failovers,
|
|
"rolling": rolling,
|
|
}
|
|
(OUT / "summary.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|