144 lines
4.9 KiB
Python
Executable File
144 lines
4.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Поочерёдный отказ каждой API-реплики и гонка одного Idempotency-Key."""
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure # noqa: E402
|
|
|
|
|
|
def compose(*args):
|
|
subprocess.check_call(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT)
|
|
|
|
|
|
def instances(samples=40):
|
|
client = Client("http://127.0.0.1:8080", timeout=2)
|
|
found = []
|
|
deadline = time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
found = []
|
|
seen = set()
|
|
try:
|
|
for _ in range(samples):
|
|
item = client.call("GET", "/health/live", schema="Health")["headers"]["X-Instance-Id"]
|
|
if item not in seen:
|
|
seen.add(item)
|
|
found.append(item)
|
|
if len(found) >= 2:
|
|
return found
|
|
except Failure:
|
|
pass
|
|
time.sleep(1)
|
|
raise SystemExit("expected at least two API instances, saw " + ",".join(found))
|
|
|
|
|
|
def restore_api():
|
|
compose("up", "-d", "--no-deps", "api")
|
|
ids = subprocess.check_output(
|
|
["docker", "compose", "--env-file", ".env", "ps", "-aq", "api"], cwd=ROOT, text=True
|
|
).split()
|
|
if ids:
|
|
subprocess.check_call(["docker", "start", *ids], cwd=ROOT)
|
|
instances()
|
|
|
|
|
|
def stop_and_measure(target):
|
|
client = Client("http://127.0.0.1:8080", timeout=2)
|
|
alice, _bob, _eve, chat = client.bootstrap()
|
|
end = time.monotonic() + 12
|
|
stats = {"ok": 0, "fail": 0, "after_ok": 0, "target_after_stop": 0, "first_recovery_s": None}
|
|
started = {}
|
|
stopped = {}
|
|
|
|
def worker():
|
|
local = Client("http://127.0.0.1:8080", timeout=2)
|
|
while time.monotonic() < end:
|
|
key = uuid.uuid4().hex
|
|
try:
|
|
response = local.message(alice, chat, "отказ " + key, key=key)
|
|
stats["ok"] += 1
|
|
mark = started.get("at")
|
|
if mark is not None and stats["first_recovery_s"] is None:
|
|
stats["first_recovery_s"] = round(time.monotonic() - mark, 3)
|
|
if stopped.get("at") is not None:
|
|
stats["after_ok"] += 1
|
|
if response["headers"].get("X-Instance-Id") == target:
|
|
stats["target_after_stop"] += 1
|
|
except Failure:
|
|
stats["fail"] += 1
|
|
|
|
threads = [threading.Thread(target=worker) for _ in range(4)]
|
|
for thread in threads:
|
|
thread.start()
|
|
time.sleep(2)
|
|
started["at"] = time.monotonic()
|
|
env = os.environ.copy()
|
|
env["TARGET_INSTANCE_ID"] = target
|
|
proc = subprocess.run([str(ROOT / "scripts" / "stop-one.sh")], cwd=ROOT, env=env, text=True, capture_output=True)
|
|
stopped["at"] = time.monotonic()
|
|
stats["hook_exit"] = proc.returncode
|
|
stats["hook_seconds"] = round(stopped["at"] - started["at"], 3)
|
|
for thread in threads:
|
|
thread.join()
|
|
stats["target"] = target
|
|
stats["successful_rps"] = round(stats["ok"] / 12, 3)
|
|
return stats
|
|
|
|
|
|
def idempotency_race():
|
|
client = Client("http://127.0.0.1:8080", timeout=5)
|
|
alice, bob, _eve, chat = client.bootstrap()
|
|
key = uuid.uuid4().hex
|
|
ids = []
|
|
errors = []
|
|
|
|
def once():
|
|
try:
|
|
ids.append(Client("http://127.0.0.1:8080", timeout=5).message(alice, chat, "гонка", key=key)["json"]["id"])
|
|
except Failure as exc:
|
|
errors.append(str(exc))
|
|
|
|
threads = [threading.Thread(target=once) for _ in range(12)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join()
|
|
page = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage")["json"]
|
|
texts = [item["text"] for item in page["items"] if item["text"] == "гонка"]
|
|
return {
|
|
"parallel_requests": 12,
|
|
"distinct_ids": len(set(ids)),
|
|
"successes": len(ids),
|
|
"errors": errors,
|
|
"history_copies": len(texts),
|
|
}
|
|
|
|
|
|
def main():
|
|
os.chdir(ROOT)
|
|
before = instances()
|
|
runs = []
|
|
for target in before:
|
|
runs.append(stop_and_measure(target))
|
|
restore_api()
|
|
race = idempotency_race()
|
|
report = {"instances_before": before, "failover": runs, "idempotency_race": race}
|
|
dest = ROOT / "evidence" / "grade4.json"
|
|
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
if any(item["hook_exit"] != 0 or item["after_ok"] < 1 or item["target_after_stop"] != 0 for item in runs):
|
|
raise SystemExit("failover measurement failed")
|
|
if race["distinct_ids"] != 1 or race["history_copies"] != 1 or race["errors"]:
|
|
raise SystemExit("idempotency race failed")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|