Files
mtusi2026lab3/scripts/verify_replicas.py
T

121 lines
4.3 KiB
Python

"""Stop each real API process under traffic; retain confirmed writes and source aliases."""
import json
import os
import subprocess
import sys
import threading
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure
def compose(*args):
return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True)
def aliases():
result = {}
for cid in compose("ps", "-q", "api").split():
alias = subprocess.check_output(
[
"docker",
"exec",
cid,
"python",
"-c",
"import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
],
text=True,
).strip()
result[cid] = alias
return result
def main():
compose("up", "-d", "--no-recreate", "--wait")
subprocess.run([sys.executable, "scripts/wait_replicas.py", "2"], cwd=ROOT, check=True)
c = Client("http://127.0.0.1:8083")
actor, _, _, chat = c.bootstrap()
initial = aliases()
runs = []
for target_container, target_alias in initial.items():
successful, errors = [], []
stop = threading.Event()
stopped_at = None
def traffic():
while not stop.is_set():
try:
response = c.message(actor, chat, "failover-proof")
successful.append(
{
"at": time.monotonic(),
"id": response["json"]["id"],
"instance": response["headers"]["X-Instance-Id"],
}
)
except Failure:
errors.append(time.monotonic())
started = time.monotonic()
thread = threading.Thread(target=traffic)
thread.start()
time.sleep(1)
stopped_at = time.monotonic()
subprocess.run(
["scripts/stop-one.sh"],
cwd=ROOT,
env={**os.environ, "TARGET_INSTANCE_ID": target_alias},
check=True,
)
time.sleep(4)
stop.set()
thread.join()
elapsed = time.monotonic() - started
after = [x for x in successful if x["at"] > stopped_at]
assert after and all(x["instance"] != target_alias for x in after[-5:])
assert min(x["at"] for x in after) - stopped_at <= 10
state = json.loads(
subprocess.check_output(["docker", "inspect", target_container], text=True)
)[0]["State"]
assert not state["Running"]
runs.append(
{
"target_container": target_container,
"target_alias": target_alias,
"successful_writes": len(successful),
"failed_attempts": len(errors),
"successful_rps": len(successful) / elapsed,
"recovery_seconds": min(x["at"] for x in after) - stopped_at,
"observed_after": sorted({x["instance"] for x in after[-5:]}),
"target_running_after_hook": state["Running"],
}
)
subprocess.run(["docker", "start", target_container], check=True, stdout=subprocess.DEVNULL)
subprocess.run([sys.executable, "scripts/wait_replicas.py", "2"], cwd=ROOT, check=True)
# Verify every acknowledged write, without assuming failed requests did not commit.
items, cursor = [], None
while True:
path = f"/api/v1/chats/{chat['id']}/messages?limit=100"
if cursor:
path += "&cursor=" + cursor
page = c.call("GET", path, actor=actor, schema="MessagePage")["json"]
items.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
break
ids = {x["id"] for x in items}
assert all(x["id"] in ids for x in successful)
runs[-1]["all_confirmed_present"] = True
report = {"real_instances_before": initial, "runs": runs, "pass": True}
(ROOT / "evidence/replica-failures.json").write_text(json.dumps(report, indent=2) + "\n")
print(json.dumps(report, indent=2))
if __name__ == "__main__":
main()