lab3: tiny fix

This commit is contained in:
yurtaev.va
2026-10-23 19:03:42 +03:00
parent 2e23dbe095
commit 3a3302c5b4
12 changed files with 418 additions and 24 deletions
+99
View File
@@ -0,0 +1,99 @@
#!/usr/bin/env python3
"""Exercise a real Swarm worker loss while reading and writing through the LB."""
import argparse
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import statistics
import subprocess
import time
import uuid
from client import Client, Failure
def tasks():
result = subprocess.check_output([
"docker", "exec", "lab3-manager", "docker", "service", "ps",
"--filter", "desired-state=running", "--format", "{{.Node}} {{.CurrentState}}",
"lab3app_api",
], text=True)
return [line.split()[0] for line in result.splitlines() if "Running" in line]
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://127.0.0.1:18080")
parser.add_argument("--worker", choices=["lab3-worker1", "lab3-worker2"], default="lab3-worker1")
parser.add_argument("--output", default="evidence/node_failover.json")
args = parser.parse_args()
client = Client(args.base_url, timeout=2)
alice, bob, _, chat = client.bootstrap()
before = tasks()
assert len(before) == 2 and args.worker in before, before
instances = {
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"]
for _ in range(40)
}
assert len(instances) == 2, instances
key = uuid.uuid4().hex
body = "concurrent " + key
with ThreadPoolExecutor(max_workers=16) as executor:
results = list(executor.map(lambda _: client.message(alice, chat, body, key=key)["json"], range(16)))
assert all(item == results[0] for item in results)
subprocess.run(["docker", "stop", "-t", "1", args.worker], check=True, stdout=subprocess.DEVNULL)
start = time.monotonic()
successes = []
errors = 0
latencies = []
moved_at = None
first_success = None
pending_key = uuid.uuid4().hex
pending_text = "node-fail " + pending_key
while time.monotonic() - start < 75:
try:
response = client.message(alice, chat, pending_text, key=pending_key)
successes.append(response["json"])
latencies.append(response["elapsed"] * 1000)
if first_success is None:
first_success = time.monotonic() - start
client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=1", actor=bob, schema="MessagePage")
pending_key = uuid.uuid4().hex
pending_text = "node-fail " + pending_key
except Failure:
errors += 1
current = tasks()
if len(current) == 2 and args.worker not in current and moved_at is None:
moved_at = time.monotonic() - start
if moved_at is not None and time.monotonic() - start > moved_at + 5:
break
time.sleep(0.15)
assert moved_at is not None, tasks()
assert successes, "no writes during node failure"
page = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage")["json"]
seen = {item["id"] for item in page["items"]}
assert results[0]["id"] in seen and all(item["id"] in seen for item in successes)
assert sum(item["text"] == body for item in page["items"]) == 1
report = {
"before_nodes": before,
"after_nodes": tasks(),
"before_instance_count": len(instances),
"concurrent_replays": len(results),
"concurrent_message_count": 1,
"successful_writes": len(successes),
"failed_attempts": errors,
"first_success_seconds": round(first_success, 3),
"replacement_running_seconds": round(moved_at, 3),
"confirmed_messages_preserved": True,
"latency_p50_ms": round(statistics.median(latencies), 3),
"latency_p95_ms": round(sorted(latencies)[int(0.95 * (len(latencies) - 1))], 3),
}
Path(args.output).write_text(json.dumps(report, indent=2) + "\n")
print(json.dumps(report, indent=2))
if __name__ == "__main__":
main()