Files
mtusi2026lab3/scripts/grade4.py
T
iDizZ d065cffb4e Добавить две реплики API за HAProxy и проверку отказа одной из них.
Общие сессии и ключ идемпотентности позволяют продолжать чтение и запись, когда одна реплика остановлена.
2026-11-06 17:31:40 +03:00

144 lines
4.9 KiB
Python
Executable File

#!/usr/bin/env python3
"""Поочерёдный отказ каждой API-реплики и гонка одного Idempotency-Key."""
import json
import os
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
def compose(*args):
subprocess.check_call(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT)
def instances(samples=40):
client = Client("http://127.0.0.1:8080", timeout=2)
found = []
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
found = []
seen = set()
try:
for _ in range(samples):
item = client.call("GET", "/health/live", schema="Health")["headers"]["X-Instance-Id"]
if item not in seen:
seen.add(item)
found.append(item)
if len(found) >= 2:
return found
except Failure:
pass
time.sleep(1)
raise SystemExit("expected at least two API instances, saw " + ",".join(found))
def restore_api():
compose("up", "-d", "--no-deps", "api")
ids = subprocess.check_output(
["docker", "compose", "--env-file", ".env", "ps", "-aq", "api"], cwd=ROOT, text=True
).split()
if ids:
subprocess.check_call(["docker", "start", *ids], cwd=ROOT)
instances()
def stop_and_measure(target):
client = Client("http://127.0.0.1:8080", timeout=2)
alice, _bob, _eve, chat = client.bootstrap()
end = time.monotonic() + 12
stats = {"ok": 0, "fail": 0, "after_ok": 0, "target_after_stop": 0, "first_recovery_s": None}
started = {}
stopped = {}
def worker():
local = Client("http://127.0.0.1:8080", timeout=2)
while time.monotonic() < end:
key = uuid.uuid4().hex
try:
response = local.message(alice, chat, "отказ " + key, key=key)
stats["ok"] += 1
mark = started.get("at")
if mark is not None and stats["first_recovery_s"] is None:
stats["first_recovery_s"] = round(time.monotonic() - mark, 3)
if stopped.get("at") is not None:
stats["after_ok"] += 1
if response["headers"].get("X-Instance-Id") == target:
stats["target_after_stop"] += 1
except Failure:
stats["fail"] += 1
threads = [threading.Thread(target=worker) for _ in range(4)]
for thread in threads:
thread.start()
time.sleep(2)
started["at"] = time.monotonic()
env = os.environ.copy()
env["TARGET_INSTANCE_ID"] = target
proc = subprocess.run([str(ROOT / "scripts" / "stop-one.sh")], cwd=ROOT, env=env, text=True, capture_output=True)
stopped["at"] = time.monotonic()
stats["hook_exit"] = proc.returncode
stats["hook_seconds"] = round(stopped["at"] - started["at"], 3)
for thread in threads:
thread.join()
stats["target"] = target
stats["successful_rps"] = round(stats["ok"] / 12, 3)
return stats
def idempotency_race():
client = Client("http://127.0.0.1:8080", timeout=5)
alice, bob, _eve, chat = client.bootstrap()
key = uuid.uuid4().hex
ids = []
errors = []
def once():
try:
ids.append(Client("http://127.0.0.1:8080", timeout=5).message(alice, chat, "гонка", key=key)["json"]["id"])
except Failure as exc:
errors.append(str(exc))
threads = [threading.Thread(target=once) for _ in range(12)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
page = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage")["json"]
texts = [item["text"] for item in page["items"] if item["text"] == "гонка"]
return {
"parallel_requests": 12,
"distinct_ids": len(set(ids)),
"successes": len(ids),
"errors": errors,
"history_copies": len(texts),
}
def main():
os.chdir(ROOT)
before = instances()
runs = []
for target in before:
runs.append(stop_and_measure(target))
restore_api()
race = idempotency_race()
report = {"instances_before": before, "failover": runs, "idempotency_race": race}
dest = ROOT / "evidence" / "grade4.json"
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(report, ensure_ascii=False, indent=2))
if any(item["hook_exit"] != 0 or item["after_ok"] < 1 or item["target_after_stop"] != 0 for item in runs):
raise SystemExit("failover measurement failed")
if race["distinct_ids"] != 1 or race["history_copies"] != 1 or race["errors"]:
raise SystemExit("idempotency race failed")
if __name__ == "__main__":
main()