Files
2026-10-23 19:29:40 +03:00

139 lines
5.6 KiB
Python

#!/usr/bin/env python3
"""Exercise a real Swarm worker loss while reading and writing through the LB."""
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import statistics
import subprocess
import time
import uuid
from urllib.parse import quote
from client import Client, Failure
def tasks():
result = subprocess.check_output([
"docker", "exec", "lab3-manager", "docker", "service", "ps",
"--filter", "desired-state=running", "--format", "{{.ID}} {{.Node}} {{.CurrentState}}",
"lab3app_api",
], text=True)
return [{"task_id": parts[0], "node": parts[1]} for line in result.splitlines()
if len(parts := line.split()) >= 3 and parts[2] == "Running"]
def history(client, actor, chat):
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
items = []
for _ in range(100):
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
items.extend(page["items"])
if page["next_cursor"] is None:
return items
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100&cursor=" + quote(page["next_cursor"], safe="")
raise AssertionError("history pagination did not terminate")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://127.0.0.1:18080")
parser.add_argument("--worker", choices=["lab3-worker1", "lab3-worker2"], default="lab3-worker1")
parser.add_argument("--output", default="evidence/node_failover.json")
args = parser.parse_args()
client = Client(args.base_url, timeout=2)
alice, bob, _, chat = client.bootstrap()
before = tasks()
assert len(before) == 2 and args.worker in {task["node"] for task in before}, before
instances = {
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"]
for _ in range(40)
}
assert len(instances) == 2, instances
key = uuid.uuid4().hex
body = "concurrent " + key
with ThreadPoolExecutor(max_workers=16) as executor:
responses = list(executor.map(lambda _: client.message(alice, chat, body, key=key), range(16)))
results = [response["json"] for response in responses]
concurrent_instances = {response["headers"]["X-Instance-Id"] for response in responses}
assert all(item == results[0] for item in results)
assert len(concurrent_instances) >= 2, concurrent_instances
start = time.monotonic()
subprocess.run(["docker", "stop", "-t", "1", args.worker], check=True, stdout=subprocess.DEVNULL)
stop_return_seconds = time.monotonic() - start
successes = {}
post_errors = 0
read_errors = 0
successful_reads = 0
latencies = []
moved_at = None
first_success = None
pending_key = uuid.uuid4().hex
pending_text = "node-fail " + pending_key
while time.monotonic() - start < 75:
try:
response = client.message(alice, chat, pending_text, key=pending_key)
assert pending_key not in successes
successes[pending_key] = response["json"]
latest_id = response["json"]["id"]
latencies.append(response["elapsed"] * 1000)
if first_success is None:
first_success = time.monotonic() - start
pending_key = uuid.uuid4().hex
pending_text = "node-fail " + pending_key
except Failure:
post_errors += 1
else:
try:
assert latest_id in {item["id"] for item in history(client, bob, chat)}
successful_reads += 1
except Failure:
read_errors += 1
current = tasks()
if len(current) == 2 and args.worker not in {task["node"] for task in current} and moved_at is None:
moved_at = time.monotonic() - start
if moved_at is not None and time.monotonic() - start > moved_at + 5:
break
time.sleep(0.15)
assert moved_at is not None, tasks()
assert successes, "no writes during node failure"
items = history(client, bob, chat)
ids = Counter(item["id"] for item in items)
texts = Counter(item["text"] for item in items)
assert ids[results[0]["id"]] == 1 and texts[body] == 1
assert all(ids[item["id"]] == 1 and texts[item["text"]] == 1 for item in successes.values())
after_instances = {
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"]
for _ in range(40)
}
assert after_instances - instances, (instances, after_instances)
report = {
"before_tasks": before,
"after_tasks": tasks(),
"before_instance_ids": sorted(instances),
"after_instance_ids": sorted(after_instances),
"concurrent_replays": len(results),
"concurrent_message_count": 1,
"concurrent_instance_ids": sorted(concurrent_instances),
"successful_writes": len(successes),
"successful_reads": successful_reads,
"failed_post_attempts": post_errors,
"failed_read_attempts": read_errors,
"docker_stop_seconds": round(stop_return_seconds, 3),
"first_success_seconds": round(first_success, 3),
"replacement_running_seconds": round(moved_at, 3),
"confirmed_messages_preserved": True,
"latency_p50_ms": round(statistics.median(latencies), 3),
"latency_p95_ms": round(sorted(latencies)[int(0.95 * (len(latencies) - 1))], 3),
}
Path(args.output).write_text(json.dumps(report, indent=2) + "\n")
print(json.dumps(report, indent=2))
if __name__ == "__main__":
main()