Files
mtusi2026lab3-fork/student-tests/check_replicas.py

119 lines
4.5 KiB
Python

from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import subprocess
import sys
from threading import Barrier, Event
import time
import uuid
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, check
REPLICAS = ("api-a", "api-b")
def compose(*args):
return subprocess.check_output(["docker", "compose", *args], text=True, timeout=45).strip()
def wait_instances(expected):
client = Client(timeout=1)
deadline = time.monotonic() + 25
seen = set()
while time.monotonic() < deadline:
try:
instance = client.call("GET", "/health/ready", schema="Health")["headers"]["X-Instance-Id"]
seen.add(instance)
if seen == expected:
return
except Failure:
pass
time.sleep(0.1)
raise AssertionError(f"Expected {expected}, observed {seen}")
def parallel_replay(actor, chat):
start = Barrier(8)
key = uuid.uuid4().hex
def write(_):
start.wait(timeout=10)
return Client().message(actor, chat, "Concurrent replay", key=key)
with ThreadPoolExecutor(max_workers=8) as workers:
results = list(workers.map(write, range(8)))
ids = {item["json"]["id"] for item in results}
instances = {item["headers"]["X-Instance-Id"] for item in results}
check(len(ids) == 1 and instances == set(REPLICAS), "parallel replay across replicas failed")
path = f"/api/v1/chats/{chat['id']}/messages"
page = Client().call("GET", path, actor=actor, schema="MessagePage")["json"]
check(sum(item["id"] in ids for item in page["items"]) == 1, "duplicate in history")
print("PASS: 8 concurrent replays through both replicas, 1 message", flush=True)
def exercise(actor, chat, target, recreate):
stop = Event()
results = []
client = Client(timeout=2)
started = time.monotonic()
def traffic():
key = uuid.uuid4().hex
while not stop.is_set():
try:
response = client.message(actor, chat, "Write " + key, key=key)
results.append((time.monotonic(), response["headers"]["X-Instance-Id"], response["json"]["id"]))
key = uuid.uuid4().hex
except Failure:
results.append((time.monotonic(), None, None))
stop.wait(0.05)
old_id = compose("ps", "-q", target)
with ThreadPoolExecutor(max_workers=1) as workers:
worker = workers.submit(traffic)
try:
time.sleep(0.5)
changed = time.monotonic()
if recreate:
compose("up", "-d", "--no-deps", "--force-recreate", target)
check(old_id != compose("ps", "-q", target), "container was not replaced")
wait_instances(set(REPLICAS))
else:
compose("stop", "--timeout", "5", target)
check(not compose("ps", "-q", "--status", "running", target), "replica is still running")
time.sleep(2.5)
finally:
stop.set()
worker.result(timeout=5)
if not recreate:
compose("start", target)
wait_instances(set(REPLICAS))
survivor = next(name for name in REPLICAS if name != target)
successful = [row for row in results if row[1] is not None]
first = next(row[0] for row in successful if row[0] >= changed and row[1] == survivor)
check(first - changed <= 10, "surviving replica took too long")
path = f"/api/v1/chats/{chat['id']}/messages?limit=100"
observed = set()
while path is not None:
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
observed.update(item["id"] for item in page["items"])
cursor = page["next_cursor"]
path = f"/api/v1/chats/{chat['id']}/messages?limit=100&cursor={cursor}" if cursor else None
check({row[2] for row in successful} <= observed, "confirmed writes lost")
action = "recreate" if recreate else "stop"
elapsed = results[-1][0] - started
print(f"PASS: {action} {target}, {len(successful) / elapsed:.1f} write/s, "
f"errors={len(results) - len(successful)}, survivor={first - changed:.3f}s", flush=True)
if __name__ == "__main__":
compose("start", *REPLICAS)
wait_instances(set(REPLICAS))
owner, _, _, chat = Client().bootstrap()
parallel_replay(owner, chat)
for name in REPLICAS:
exercise(owner, chat, name, recreate=False)
for name in REPLICAS:
exercise(owner, chat, name, recreate=True)