#!/usr/bin/env python3 """Поочерёдный отказ каждой API-реплики и гонка одного Idempotency-Key.""" import json import os import subprocess import sys import threading import time import uuid from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure # noqa: E402 def compose(*args): subprocess.check_call(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT) def instances(samples=40): client = Client("http://127.0.0.1:8080", timeout=2) found = [] deadline = time.monotonic() + 60 while time.monotonic() < deadline: found = [] seen = set() try: for _ in range(samples): item = client.call("GET", "/health/live", schema="Health")["headers"]["X-Instance-Id"] if item not in seen: seen.add(item) found.append(item) if len(found) >= 2: return found except Failure: pass time.sleep(1) raise SystemExit("expected at least two API instances, saw " + ",".join(found)) def restore_api(): compose("up", "-d", "--no-deps", "api") ids = subprocess.check_output( ["docker", "compose", "--env-file", ".env", "ps", "-aq", "api"], cwd=ROOT, text=True ).split() if ids: subprocess.check_call(["docker", "start", *ids], cwd=ROOT) instances() def stop_and_measure(target): client = Client("http://127.0.0.1:8080", timeout=2) alice, _bob, _eve, chat = client.bootstrap() end = time.monotonic() + 12 stats = {"ok": 0, "fail": 0, "after_ok": 0, "target_after_stop": 0, "first_recovery_s": None} started = {} stopped = {} def worker(): local = Client("http://127.0.0.1:8080", timeout=2) while time.monotonic() < end: key = uuid.uuid4().hex try: response = local.message(alice, chat, "отказ " + key, key=key) stats["ok"] += 1 mark = started.get("at") if mark is not None and stats["first_recovery_s"] is None: stats["first_recovery_s"] = round(time.monotonic() - mark, 3) if stopped.get("at") is not None: stats["after_ok"] += 1 if response["headers"].get("X-Instance-Id") == target: stats["target_after_stop"] += 1 except Failure: stats["fail"] += 1 threads = [threading.Thread(target=worker) for _ in range(4)] for thread in threads: thread.start() time.sleep(2) started["at"] = time.monotonic() env = os.environ.copy() env["TARGET_INSTANCE_ID"] = target proc = subprocess.run([str(ROOT / "scripts" / "stop-one.sh")], cwd=ROOT, env=env, text=True, capture_output=True) stopped["at"] = time.monotonic() stats["hook_exit"] = proc.returncode stats["hook_seconds"] = round(stopped["at"] - started["at"], 3) for thread in threads: thread.join() stats["target"] = target stats["successful_rps"] = round(stats["ok"] / 12, 3) return stats def idempotency_race(): client = Client("http://127.0.0.1:8080", timeout=5) alice, bob, _eve, chat = client.bootstrap() key = uuid.uuid4().hex ids = [] errors = [] def once(): try: ids.append(Client("http://127.0.0.1:8080", timeout=5).message(alice, chat, "гонка", key=key)["json"]["id"]) except Failure as exc: errors.append(str(exc)) threads = [threading.Thread(target=once) for _ in range(12)] for thread in threads: thread.start() for thread in threads: thread.join() page = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage")["json"] texts = [item["text"] for item in page["items"] if item["text"] == "гонка"] return { "parallel_requests": 12, "distinct_ids": len(set(ids)), "successes": len(ids), "errors": errors, "history_copies": len(texts), } def main(): os.chdir(ROOT) before = instances() runs = [] for target in before: runs.append(stop_and_measure(target)) restore_api() race = idempotency_race() report = {"instances_before": before, "failover": runs, "idempotency_race": race} dest = ROOT / "evidence" / "grade4.json" dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") print(json.dumps(report, ensure_ascii=False, indent=2)) if any(item["hook_exit"] != 0 or item["after_ok"] < 1 or item["target_after_stop"] != 0 for item in runs): raise SystemExit("failover measurement failed") if race["distinct_ids"] != 1 or race["history_copies"] != 1 or race["errors"]: raise SystemExit("idempotency race failed") if __name__ == "__main__": main()