forked from mtusi2026/mtusi2026lab3
139 lines
5.6 KiB
Python
139 lines
5.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Exercise a real Swarm worker loss while reading and writing through the LB."""
|
|
|
|
import argparse
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
from pathlib import Path
|
|
import statistics
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
from urllib.parse import quote
|
|
|
|
from client import Client, Failure
|
|
|
|
|
|
def tasks():
|
|
result = subprocess.check_output([
|
|
"docker", "exec", "lab3-manager", "docker", "service", "ps",
|
|
"--filter", "desired-state=running", "--format", "{{.ID}} {{.Node}} {{.CurrentState}}",
|
|
"lab3app_api",
|
|
], text=True)
|
|
return [{"task_id": parts[0], "node": parts[1]} for line in result.splitlines()
|
|
if len(parts := line.split()) >= 3 and parts[2] == "Running"]
|
|
|
|
|
|
def history(client, actor, chat):
|
|
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
|
|
items = []
|
|
for _ in range(100):
|
|
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
|
items.extend(page["items"])
|
|
if page["next_cursor"] is None:
|
|
return items
|
|
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100&cursor=" + quote(page["next_cursor"], safe="")
|
|
raise AssertionError("history pagination did not terminate")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--base-url", default="http://127.0.0.1:18080")
|
|
parser.add_argument("--worker", choices=["lab3-worker1", "lab3-worker2"], default="lab3-worker1")
|
|
parser.add_argument("--output", default="evidence/node_failover.json")
|
|
args = parser.parse_args()
|
|
client = Client(args.base_url, timeout=2)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
before = tasks()
|
|
assert len(before) == 2 and args.worker in {task["node"] for task in before}, before
|
|
instances = {
|
|
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"]
|
|
for _ in range(40)
|
|
}
|
|
assert len(instances) == 2, instances
|
|
|
|
key = uuid.uuid4().hex
|
|
body = "concurrent " + key
|
|
with ThreadPoolExecutor(max_workers=16) as executor:
|
|
responses = list(executor.map(lambda _: client.message(alice, chat, body, key=key), range(16)))
|
|
results = [response["json"] for response in responses]
|
|
concurrent_instances = {response["headers"]["X-Instance-Id"] for response in responses}
|
|
assert all(item == results[0] for item in results)
|
|
assert len(concurrent_instances) >= 2, concurrent_instances
|
|
|
|
start = time.monotonic()
|
|
subprocess.run(["docker", "stop", "-t", "1", args.worker], check=True, stdout=subprocess.DEVNULL)
|
|
stop_return_seconds = time.monotonic() - start
|
|
successes = {}
|
|
post_errors = 0
|
|
read_errors = 0
|
|
successful_reads = 0
|
|
latencies = []
|
|
moved_at = None
|
|
first_success = None
|
|
pending_key = uuid.uuid4().hex
|
|
pending_text = "node-fail " + pending_key
|
|
while time.monotonic() - start < 75:
|
|
try:
|
|
response = client.message(alice, chat, pending_text, key=pending_key)
|
|
assert pending_key not in successes
|
|
successes[pending_key] = response["json"]
|
|
latest_id = response["json"]["id"]
|
|
latencies.append(response["elapsed"] * 1000)
|
|
if first_success is None:
|
|
first_success = time.monotonic() - start
|
|
pending_key = uuid.uuid4().hex
|
|
pending_text = "node-fail " + pending_key
|
|
except Failure:
|
|
post_errors += 1
|
|
else:
|
|
try:
|
|
assert latest_id in {item["id"] for item in history(client, bob, chat)}
|
|
successful_reads += 1
|
|
except Failure:
|
|
read_errors += 1
|
|
current = tasks()
|
|
if len(current) == 2 and args.worker not in {task["node"] for task in current} and moved_at is None:
|
|
moved_at = time.monotonic() - start
|
|
if moved_at is not None and time.monotonic() - start > moved_at + 5:
|
|
break
|
|
time.sleep(0.15)
|
|
assert moved_at is not None, tasks()
|
|
assert successes, "no writes during node failure"
|
|
items = history(client, bob, chat)
|
|
ids = Counter(item["id"] for item in items)
|
|
texts = Counter(item["text"] for item in items)
|
|
assert ids[results[0]["id"]] == 1 and texts[body] == 1
|
|
assert all(ids[item["id"]] == 1 and texts[item["text"]] == 1 for item in successes.values())
|
|
after_instances = {
|
|
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"]
|
|
for _ in range(40)
|
|
}
|
|
assert after_instances - instances, (instances, after_instances)
|
|
report = {
|
|
"before_tasks": before,
|
|
"after_tasks": tasks(),
|
|
"before_instance_ids": sorted(instances),
|
|
"after_instance_ids": sorted(after_instances),
|
|
"concurrent_replays": len(results),
|
|
"concurrent_message_count": 1,
|
|
"concurrent_instance_ids": sorted(concurrent_instances),
|
|
"successful_writes": len(successes),
|
|
"successful_reads": successful_reads,
|
|
"failed_post_attempts": post_errors,
|
|
"failed_read_attempts": read_errors,
|
|
"docker_stop_seconds": round(stop_return_seconds, 3),
|
|
"first_success_seconds": round(first_success, 3),
|
|
"replacement_running_seconds": round(moved_at, 3),
|
|
"confirmed_messages_preserved": True,
|
|
"latency_p50_ms": round(statistics.median(latencies), 3),
|
|
"latency_p95_ms": round(sorted(latencies)[int(0.95 * (len(latencies) - 1))], 3),
|
|
}
|
|
Path(args.output).write_text(json.dumps(report, indent=2) + "\n")
|
|
print(json.dumps(report, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|