Files
mtusi2026lab3-yurtaev/tests/node_failover.py
T
2026-10-23 19:03:42 +03:00

100 lines
3.9 KiB
Python

#!/usr/bin/env python3
"""Exercise a real Swarm worker loss while reading and writing through the LB."""
import argparse
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import statistics
import subprocess
import time
import uuid
from client import Client, Failure
def tasks():
result = subprocess.check_output([
"docker", "exec", "lab3-manager", "docker", "service", "ps",
"--filter", "desired-state=running", "--format", "{{.Node}} {{.CurrentState}}",
"lab3app_api",
], text=True)
return [line.split()[0] for line in result.splitlines() if "Running" in line]
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://127.0.0.1:18080")
parser.add_argument("--worker", choices=["lab3-worker1", "lab3-worker2"], default="lab3-worker1")
parser.add_argument("--output", default="evidence/node_failover.json")
args = parser.parse_args()
client = Client(args.base_url, timeout=2)
alice, bob, _, chat = client.bootstrap()
before = tasks()
assert len(before) == 2 and args.worker in before, before
instances = {
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"]
for _ in range(40)
}
assert len(instances) == 2, instances
key = uuid.uuid4().hex
body = "concurrent " + key
with ThreadPoolExecutor(max_workers=16) as executor:
results = list(executor.map(lambda _: client.message(alice, chat, body, key=key)["json"], range(16)))
assert all(item == results[0] for item in results)
subprocess.run(["docker", "stop", "-t", "1", args.worker], check=True, stdout=subprocess.DEVNULL)
start = time.monotonic()
successes = []
errors = 0
latencies = []
moved_at = None
first_success = None
pending_key = uuid.uuid4().hex
pending_text = "node-fail " + pending_key
while time.monotonic() - start < 75:
try:
response = client.message(alice, chat, pending_text, key=pending_key)
successes.append(response["json"])
latencies.append(response["elapsed"] * 1000)
if first_success is None:
first_success = time.monotonic() - start
client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=1", actor=bob, schema="MessagePage")
pending_key = uuid.uuid4().hex
pending_text = "node-fail " + pending_key
except Failure:
errors += 1
current = tasks()
if len(current) == 2 and args.worker not in current and moved_at is None:
moved_at = time.monotonic() - start
if moved_at is not None and time.monotonic() - start > moved_at + 5:
break
time.sleep(0.15)
assert moved_at is not None, tasks()
assert successes, "no writes during node failure"
page = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage")["json"]
seen = {item["id"] for item in page["items"]}
assert results[0]["id"] in seen and all(item["id"] in seen for item in successes)
assert sum(item["text"] == body for item in page["items"]) == 1
report = {
"before_nodes": before,
"after_nodes": tasks(),
"before_instance_count": len(instances),
"concurrent_replays": len(results),
"concurrent_message_count": 1,
"successful_writes": len(successes),
"failed_attempts": errors,
"first_success_seconds": round(first_success, 3),
"replacement_running_seconds": round(moved_at, 3),
"confirmed_messages_preserved": True,
"latency_p50_ms": round(statistics.median(latencies), 3),
"latency_p95_ms": round(sorted(latencies)[int(0.95 * (len(latencies) - 1))], 3),
}
Path(args.output).write_text(json.dumps(report, indent=2) + "\n")
print(json.dumps(report, indent=2))
if __name__ == "__main__":
main()