forked from mtusi2026/mtusi2026lab3
119 lines
4.5 KiB
Python
119 lines
4.5 KiB
Python
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
from threading import Barrier, Event
|
|
import time
|
|
import uuid
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
|
from client import Client, Failure, check
|
|
|
|
REPLICAS = ("api-a", "api-b")
|
|
|
|
|
|
def compose(*args):
|
|
return subprocess.check_output(["docker", "compose", *args], text=True, timeout=45).strip()
|
|
|
|
|
|
def wait_instances(expected):
|
|
client = Client(timeout=1)
|
|
deadline = time.monotonic() + 25
|
|
seen = set()
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
instance = client.call("GET", "/health/ready", schema="Health")["headers"]["X-Instance-Id"]
|
|
seen.add(instance)
|
|
if seen == expected:
|
|
return
|
|
except Failure:
|
|
pass
|
|
time.sleep(0.1)
|
|
raise AssertionError(f"Expected {expected}, observed {seen}")
|
|
|
|
|
|
def parallel_replay(actor, chat):
|
|
start = Barrier(8)
|
|
key = uuid.uuid4().hex
|
|
|
|
def write(_):
|
|
start.wait(timeout=10)
|
|
return Client().message(actor, chat, "Concurrent replay", key=key)
|
|
|
|
with ThreadPoolExecutor(max_workers=8) as workers:
|
|
results = list(workers.map(write, range(8)))
|
|
ids = {item["json"]["id"] for item in results}
|
|
instances = {item["headers"]["X-Instance-Id"] for item in results}
|
|
check(len(ids) == 1 and instances == set(REPLICAS), "parallel replay across replicas failed")
|
|
path = f"/api/v1/chats/{chat['id']}/messages"
|
|
page = Client().call("GET", path, actor=actor, schema="MessagePage")["json"]
|
|
check(sum(item["id"] in ids for item in page["items"]) == 1, "duplicate in history")
|
|
print("PASS: 8 concurrent replays through both replicas, 1 message", flush=True)
|
|
|
|
|
|
def exercise(actor, chat, target, recreate):
|
|
stop = Event()
|
|
results = []
|
|
client = Client(timeout=2)
|
|
started = time.monotonic()
|
|
|
|
def traffic():
|
|
key = uuid.uuid4().hex
|
|
while not stop.is_set():
|
|
try:
|
|
response = client.message(actor, chat, "Write " + key, key=key)
|
|
results.append((time.monotonic(), response["headers"]["X-Instance-Id"], response["json"]["id"]))
|
|
key = uuid.uuid4().hex
|
|
except Failure:
|
|
results.append((time.monotonic(), None, None))
|
|
stop.wait(0.05)
|
|
|
|
old_id = compose("ps", "-q", target)
|
|
with ThreadPoolExecutor(max_workers=1) as workers:
|
|
worker = workers.submit(traffic)
|
|
try:
|
|
time.sleep(0.5)
|
|
changed = time.monotonic()
|
|
if recreate:
|
|
compose("up", "-d", "--no-deps", "--force-recreate", target)
|
|
check(old_id != compose("ps", "-q", target), "container was not replaced")
|
|
wait_instances(set(REPLICAS))
|
|
else:
|
|
compose("stop", "--timeout", "5", target)
|
|
check(not compose("ps", "-q", "--status", "running", target), "replica is still running")
|
|
time.sleep(2.5)
|
|
finally:
|
|
stop.set()
|
|
worker.result(timeout=5)
|
|
if not recreate:
|
|
compose("start", target)
|
|
wait_instances(set(REPLICAS))
|
|
|
|
survivor = next(name for name in REPLICAS if name != target)
|
|
successful = [row for row in results if row[1] is not None]
|
|
first = next(row[0] for row in successful if row[0] >= changed and row[1] == survivor)
|
|
check(first - changed <= 10, "surviving replica took too long")
|
|
path = f"/api/v1/chats/{chat['id']}/messages?limit=100"
|
|
observed = set()
|
|
while path is not None:
|
|
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
|
observed.update(item["id"] for item in page["items"])
|
|
cursor = page["next_cursor"]
|
|
path = f"/api/v1/chats/{chat['id']}/messages?limit=100&cursor={cursor}" if cursor else None
|
|
check({row[2] for row in successful} <= observed, "confirmed writes lost")
|
|
action = "recreate" if recreate else "stop"
|
|
elapsed = results[-1][0] - started
|
|
print(f"PASS: {action} {target}, {len(successful) / elapsed:.1f} write/s, "
|
|
f"errors={len(results) - len(successful)}, survivor={first - changed:.3f}s", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
compose("start", *REPLICAS)
|
|
wait_instances(set(REPLICAS))
|
|
owner, _, _, chat = Client().bootstrap()
|
|
parallel_replay(owner, chat)
|
|
for name in REPLICAS:
|
|
exercise(owner, chat, name, recreate=False)
|
|
for name in REPLICAS:
|
|
exercise(owner, chat, name, recreate=True)
|