lab3: tiny fix
This commit is contained in:
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise a real Swarm worker loss while reading and writing through the LB."""
|
||||
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from client import Client, Failure
|
||||
|
||||
|
||||
def tasks():
|
||||
result = subprocess.check_output([
|
||||
"docker", "exec", "lab3-manager", "docker", "service", "ps",
|
||||
"--filter", "desired-state=running", "--format", "{{.Node}} {{.CurrentState}}",
|
||||
"lab3app_api",
|
||||
], text=True)
|
||||
return [line.split()[0] for line in result.splitlines() if "Running" in line]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:18080")
|
||||
parser.add_argument("--worker", choices=["lab3-worker1", "lab3-worker2"], default="lab3-worker1")
|
||||
parser.add_argument("--output", default="evidence/node_failover.json")
|
||||
args = parser.parse_args()
|
||||
client = Client(args.base_url, timeout=2)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
before = tasks()
|
||||
assert len(before) == 2 and args.worker in before, before
|
||||
instances = {
|
||||
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"]
|
||||
for _ in range(40)
|
||||
}
|
||||
assert len(instances) == 2, instances
|
||||
|
||||
key = uuid.uuid4().hex
|
||||
body = "concurrent " + key
|
||||
with ThreadPoolExecutor(max_workers=16) as executor:
|
||||
results = list(executor.map(lambda _: client.message(alice, chat, body, key=key)["json"], range(16)))
|
||||
assert all(item == results[0] for item in results)
|
||||
|
||||
subprocess.run(["docker", "stop", "-t", "1", args.worker], check=True, stdout=subprocess.DEVNULL)
|
||||
start = time.monotonic()
|
||||
successes = []
|
||||
errors = 0
|
||||
latencies = []
|
||||
moved_at = None
|
||||
first_success = None
|
||||
pending_key = uuid.uuid4().hex
|
||||
pending_text = "node-fail " + pending_key
|
||||
while time.monotonic() - start < 75:
|
||||
try:
|
||||
response = client.message(alice, chat, pending_text, key=pending_key)
|
||||
successes.append(response["json"])
|
||||
latencies.append(response["elapsed"] * 1000)
|
||||
if first_success is None:
|
||||
first_success = time.monotonic() - start
|
||||
client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=1", actor=bob, schema="MessagePage")
|
||||
pending_key = uuid.uuid4().hex
|
||||
pending_text = "node-fail " + pending_key
|
||||
except Failure:
|
||||
errors += 1
|
||||
current = tasks()
|
||||
if len(current) == 2 and args.worker not in current and moved_at is None:
|
||||
moved_at = time.monotonic() - start
|
||||
if moved_at is not None and time.monotonic() - start > moved_at + 5:
|
||||
break
|
||||
time.sleep(0.15)
|
||||
assert moved_at is not None, tasks()
|
||||
assert successes, "no writes during node failure"
|
||||
page = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage")["json"]
|
||||
seen = {item["id"] for item in page["items"]}
|
||||
assert results[0]["id"] in seen and all(item["id"] in seen for item in successes)
|
||||
assert sum(item["text"] == body for item in page["items"]) == 1
|
||||
report = {
|
||||
"before_nodes": before,
|
||||
"after_nodes": tasks(),
|
||||
"before_instance_count": len(instances),
|
||||
"concurrent_replays": len(results),
|
||||
"concurrent_message_count": 1,
|
||||
"successful_writes": len(successes),
|
||||
"failed_attempts": errors,
|
||||
"first_success_seconds": round(first_success, 3),
|
||||
"replacement_running_seconds": round(moved_at, 3),
|
||||
"confirmed_messages_preserved": True,
|
||||
"latency_p50_ms": round(statistics.median(latencies), 3),
|
||||
"latency_p95_ms": round(sorted(latencies)[int(0.95 * (len(latencies) - 1))], 3),
|
||||
}
|
||||
Path(args.output).write_text(json.dumps(report, indent=2) + "\n")
|
||||
print(json.dumps(report, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user