Files
mtusi2026lab3/scripts/grade4_evidence.py
T
dawuduk 99ce1b1391 Измерить отказ каждой реплики и цену добавления процессов.
Один и тот же профиль нагрузки сравнивает 1, 2 и 4 реплики при фиксированном и растущем бюджете CPU.
2026-11-07 12:46:37 +03:00

230 lines
7.3 KiB
Python

#!/usr/bin/env python3
"""Grade 4: parallel idempotency, stop each API replica, rolling replacement under load."""
import json
import os
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
OUT = ROOT / "evidence" / "grade4"
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
def compose(*args, check=True):
cmd = ["docker", "compose"]
if (ROOT / ".env").is_file():
cmd.extend(["--env-file", str(ROOT / ".env")])
cmd.extend(args)
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
if check and proc.returncode != 0:
raise SystemExit("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or proc.stdout)[-800:]))
return proc
def bash_argv(script: Path):
if os.name != "nt":
return [str(script)]
git_bash = Path(r"C:\Program Files\Git\bin\bash.exe")
if git_bash.is_file():
return [str(git_bash), str(script)]
return ["bash", str(script)]
def hook_cmd():
return bash_argv(ROOT / "scripts" / "stop-one.sh")
def wait_ready(seconds=60):
deadline = time.monotonic() + seconds
last = None
while time.monotonic() < deadline:
try:
Client(BASE, timeout=3).call("GET", "/health/ready", schema="Health")
return
except Failure as exc:
last = exc
time.sleep(0.4)
raise SystemExit("API did not become ready: %s" % last)
def instance_ids(actor, samples=24):
found = []
client = Client(BASE, timeout=3)
for _ in range(samples):
found.append(client.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"]["X-Instance-Id"])
return sorted(set(found))
def parallel_idempotency():
client = Client(BASE, timeout=20)
alice, _bob, _eve, chat = client.bootstrap()
key = uuid.uuid4().hex
text = "parallel-" + key
ids = []
errors = []
barrier = threading.Barrier(16)
def one():
local = Client(BASE, timeout=20)
barrier.wait()
try:
ids.append(local.message(alice, chat, text, key=key)["json"]["id"])
except Exception as exc:
errors.append(type(exc).__name__)
threads = [threading.Thread(target=one) for _ in range(16)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
unique = sorted(set(ids))
report = {
"workers": 16,
"successes": len(ids),
"errors": len(errors),
"distinct_message_ids": len(unique),
}
if len(unique) != 1 or errors:
raise SystemExit("parallel idempotency failed: %s" % report)
page = client.call(
"GET",
"/api/v1/chats/%s/messages?limit=100" % chat["id"],
actor=alice,
schema="MessagePage",
)["json"]
copies = [item for item in page["items"] if item["text"] == text]
report["history_copies"] = len(copies)
if len(copies) != 1:
raise SystemExit("history has %s copies" % len(copies))
return report
def failover_one(actor, chat, instance_id):
key = "failover-" + uuid.uuid4().hex
text = "during-" + key
successes = []
failures = 0
started_box = {}
def traffic():
nonlocal failures
local = Client(BASE, timeout=2)
while "t0" not in started_box:
time.sleep(0.01)
deadline = started_box["t0"] + 12
recovered = False
while time.monotonic() < deadline:
try:
if not recovered:
response = local.message(actor, chat, text, key=key)
recovered = True
else:
response = local.message(actor, chat, "after-" + uuid.uuid4().hex)
successes.append((time.monotonic(), response["headers"]["X-Instance-Id"], response["json"]["id"]))
except Failure:
failures += 1
time.sleep(0.05)
worker = threading.Thread(target=traffic)
worker.start()
time.sleep(0.2)
started = time.monotonic()
started_box["t0"] = started
env = os.environ.copy()
env["TARGET_INSTANCE_ID"] = instance_id
proc = subprocess.run(hook_cmd(), cwd=ROOT, env=env, capture_output=True, text=True)
hook_s = time.monotonic() - started
worker.join()
after = [item for item in successes if item[0] >= started]
recovery = None
if after:
recovery = round(after[0][0] - started, 3)
tail = [item[1] for item in after[-5:]]
return {
"target": instance_id,
"hook_exit": proc.returncode,
"hook_seconds": round(hook_s, 3),
"hook_stderr": (proc.stderr or "")[-300:],
"failed_attempts": failures,
"successes_after_hook_start": len(after),
"first_recovery_seconds": recovery,
"tail_instance_ids": tail,
"target_still_in_tail": instance_id in tail,
}
def restore_and_wait(actor):
compose("start", "api")
deadline = time.monotonic() + 40
while time.monotonic() < deadline:
try:
if len(instance_ids(actor, samples=16)) >= 2:
return
except Failure:
pass
time.sleep(0.5)
raise SystemExit("second replica did not return")
def rolling_under_load():
out = OUT / "rolling-load.json"
load = subprocess.Popen(
[sys.executable, "-X", "utf8", str(ROOT / "tests" / "load.py"),
"--base-url", BASE, "--duration", "50", "--concurrency", "4", "--output", str(out)],
cwd=ROOT,
)
time.sleep(3)
cmd = bash_argv(ROOT / "scripts" / "rolling-update.sh")
started = time.monotonic()
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
elapsed = round(time.monotonic() - started, 3)
load.wait(timeout=90)
metrics = json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}
if proc.returncode != 0:
raise SystemExit("rolling update failed: %s" % (proc.stderr or proc.stdout)[-800:])
return {
"rolling_seconds": elapsed,
"rolling_stdout": (proc.stdout or "")[-400:],
"load": metrics,
}
def main():
OUT.mkdir(parents=True, exist_ok=True)
wait_ready()
client = Client(BASE, timeout=10)
alice, _bob, _eve, chat = client.bootstrap()
before = instance_ids(alice)
if len(before) < 2:
raise SystemExit("need 2 instance ids before grade-4 runs, saw %s" % before)
idem = parallel_idempotency()
failovers = []
for instance_id in before:
item = failover_one(alice, chat, instance_id)
failovers.append(item)
recovery = item["first_recovery_seconds"]
if item["hook_exit"] != 0 or item["target_still_in_tail"] or recovery is None or recovery > 10 or item["successes_after_hook_start"] < 5:
raise SystemExit("failover criteria failed: %s" % item)
restore_and_wait(alice)
rolling = rolling_under_load()
report = {
"instances_before": before,
"parallel_idempotency": idem,
"failovers": failovers,
"rolling": rolling,
}
(OUT / "summary.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()