Files
mtusi2026lab3/scripts/check_failover.py
T

103 lines
5.0 KiB
Python

import json
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, arguments, check, run
from resilience import history
COMPOSE = ["docker", "compose", "--project-name", "highload-lab3", "--env-file", ".env", "-f", "compose.yml"]
def inspect(container):
# Deliberately inspect only public identity/state, not credentials or env.
raw = subprocess.check_output(["docker", "inspect", "--format",
'{{json .Config.Hostname}} {{json .State.Status}}', container], text=True)
hostname, state = raw.strip().split()
return json.loads(hostname), json.loads(state)
def restore():
subprocess.run(COMPOSE + ["up", "-d", "--wait", "--no-deps", "api"], cwd=ROOT, check=True)
def main():
args = arguments(__doc__)
c = Client(args.base_url, args.ca_file, timeout=2)
containers = subprocess.check_output(COMPOSE + ["ps", "-q", "api"], cwd=ROOT, text=True).split()
check(len(containers) >= 2, "need at least two running API containers")
targets = [(container, inspect(container)[0]) for container in containers]
alice, bob, _, chat = c.bootstrap()
records = []
for container, target in targets:
# Read through the LB using one existing session until all replicas rejoin.
seen = set()
deadline = time.monotonic() + 10
while len(seen) < len(targets) and time.monotonic() < deadline:
seen.add(c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"])
time.sleep(0.05)
check(target in seen and len(seen) >= 2, "restored replica is not serving through HAProxy")
confirmed_key = uuid.uuid4().hex
confirmed_text = "Before failure " + confirmed_key
original = c.message(alice, chat, confirmed_text, key=confirmed_key)["json"]
env = dict(os.environ, TARGET_INSTANCE_ID=target)
started = time.monotonic()
proc = subprocess.Popen([str(ROOT / "scripts" / "stop-one.sh")], cwd=ROOT, env=env,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
successes, last_instances = [], []
write_errors = read_errors = reads = 0
first_write = first_read = None
key = uuid.uuid4().hex
try:
while time.monotonic() - started < 15:
check(proc.poll() in (None, 0), "stop-one hook failed")
try:
result = c.message(alice, chat, "During failure " + key, key=key)
successes.append(result["json"])
if proc.poll() == 0:
last_instances.append(result["headers"]["X-Instance-Id"])
if first_write is None:
first_write = time.monotonic() - started
key = uuid.uuid4().hex
except Failure:
write_errors += 1 # Retry the SAME key and text on the next iteration.
try:
c.call("GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=bob, schema="MessagePage")
reads += 1
if proc.poll() == 0 and first_read is None:
first_read = time.monotonic() - started
except Failure:
read_errors += 1
time.sleep(0.1)
check(proc.poll() == 0, "stop-one hook did not finish")
check(inspect(container)[1] == "exited", "target container did not stay stopped")
check(first_write is not None and first_write <= 10, "writes did not recover within 10 seconds")
check(first_read is not None and first_read <= 10, "reads did not recover within 10 seconds")
check(len(last_instances) >= 5 and target not in last_instances[-5:], "stopped replica still serves responses")
check(c.message(alice, chat, confirmed_text, key=confirmed_key)["json"] == original,
"replay after switching replicas changed a confirmed message")
messages = history(c, bob, chat)
check(all(message in messages for message in [original] + successes), "confirmed message lost")
texts = [message["text"] for message in messages]
check(len(texts) == len(set(texts)), "uncertain retry created a duplicate")
record = {"stopped_instance": target, "container_state": "exited", "successful_writes": len(successes),
"successful_reads": reads, "write_errors": write_errors, "read_errors": read_errors,
"first_write_seconds": round(first_write, 3), "first_read_seconds": round(first_read, 3)}
records.append(record)
print(json.dumps(record), flush=True)
finally:
if proc.poll() is None:
proc.terminate()
proc.wait(timeout=5)
restore()
print("PASS: each API replica stopped in turn; reads, writes, sessions and confirmed messages preserved")
if __name__ == "__main__":
run(main)