forked from mtusi2026/mtusi2026lab3
Добавить две реплики API за HAProxy и проверку отказа одной из них.
Общие сессии и ключ идемпотентности позволяют продолжать чтение и запись, когда одна реплика остановлена.
This commit is contained in:
Executable
+143
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Поочерёдный отказ каждой API-реплики и гонка одного Idempotency-Key."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure # noqa: E402
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT)
|
||||
|
||||
|
||||
def instances(samples=40):
|
||||
client = Client("http://127.0.0.1:8080", timeout=2)
|
||||
found = []
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
found = []
|
||||
seen = set()
|
||||
try:
|
||||
for _ in range(samples):
|
||||
item = client.call("GET", "/health/live", schema="Health")["headers"]["X-Instance-Id"]
|
||||
if item not in seen:
|
||||
seen.add(item)
|
||||
found.append(item)
|
||||
if len(found) >= 2:
|
||||
return found
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise SystemExit("expected at least two API instances, saw " + ",".join(found))
|
||||
|
||||
|
||||
def restore_api():
|
||||
compose("up", "-d", "--no-deps", "api")
|
||||
ids = subprocess.check_output(
|
||||
["docker", "compose", "--env-file", ".env", "ps", "-aq", "api"], cwd=ROOT, text=True
|
||||
).split()
|
||||
if ids:
|
||||
subprocess.check_call(["docker", "start", *ids], cwd=ROOT)
|
||||
instances()
|
||||
|
||||
|
||||
def stop_and_measure(target):
|
||||
client = Client("http://127.0.0.1:8080", timeout=2)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
end = time.monotonic() + 12
|
||||
stats = {"ok": 0, "fail": 0, "after_ok": 0, "target_after_stop": 0, "first_recovery_s": None}
|
||||
started = {}
|
||||
stopped = {}
|
||||
|
||||
def worker():
|
||||
local = Client("http://127.0.0.1:8080", timeout=2)
|
||||
while time.monotonic() < end:
|
||||
key = uuid.uuid4().hex
|
||||
try:
|
||||
response = local.message(alice, chat, "отказ " + key, key=key)
|
||||
stats["ok"] += 1
|
||||
mark = started.get("at")
|
||||
if mark is not None and stats["first_recovery_s"] is None:
|
||||
stats["first_recovery_s"] = round(time.monotonic() - mark, 3)
|
||||
if stopped.get("at") is not None:
|
||||
stats["after_ok"] += 1
|
||||
if response["headers"].get("X-Instance-Id") == target:
|
||||
stats["target_after_stop"] += 1
|
||||
except Failure:
|
||||
stats["fail"] += 1
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(4)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
time.sleep(2)
|
||||
started["at"] = time.monotonic()
|
||||
env = os.environ.copy()
|
||||
env["TARGET_INSTANCE_ID"] = target
|
||||
proc = subprocess.run([str(ROOT / "scripts" / "stop-one.sh")], cwd=ROOT, env=env, text=True, capture_output=True)
|
||||
stopped["at"] = time.monotonic()
|
||||
stats["hook_exit"] = proc.returncode
|
||||
stats["hook_seconds"] = round(stopped["at"] - started["at"], 3)
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
stats["target"] = target
|
||||
stats["successful_rps"] = round(stats["ok"] / 12, 3)
|
||||
return stats
|
||||
|
||||
|
||||
def idempotency_race():
|
||||
client = Client("http://127.0.0.1:8080", timeout=5)
|
||||
alice, bob, _eve, chat = client.bootstrap()
|
||||
key = uuid.uuid4().hex
|
||||
ids = []
|
||||
errors = []
|
||||
|
||||
def once():
|
||||
try:
|
||||
ids.append(Client("http://127.0.0.1:8080", timeout=5).message(alice, chat, "гонка", key=key)["json"]["id"])
|
||||
except Failure as exc:
|
||||
errors.append(str(exc))
|
||||
|
||||
threads = [threading.Thread(target=once) for _ in range(12)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
page = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage")["json"]
|
||||
texts = [item["text"] for item in page["items"] if item["text"] == "гонка"]
|
||||
return {
|
||||
"parallel_requests": 12,
|
||||
"distinct_ids": len(set(ids)),
|
||||
"successes": len(ids),
|
||||
"errors": errors,
|
||||
"history_copies": len(texts),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
os.chdir(ROOT)
|
||||
before = instances()
|
||||
runs = []
|
||||
for target in before:
|
||||
runs.append(stop_and_measure(target))
|
||||
restore_api()
|
||||
race = idempotency_race()
|
||||
report = {"instances_before": before, "failover": runs, "idempotency_race": race}
|
||||
dest = ROOT / "evidence" / "grade4.json"
|
||||
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if any(item["hook_exit"] != 0 or item["after_ok"] < 1 or item["target_after_stop"] != 0 for item in runs):
|
||||
raise SystemExit("failover measurement failed")
|
||||
if race["distinct_ids"] != 1 or race["history_copies"] != 1 or race["errors"]:
|
||||
raise SystemExit("idempotency race failed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user