import json import os import subprocess import sys import time import uuid from pathlib import Path from rolling_update import identity, stats ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure, arguments, check, run from resilience import history COMPOSE = ["docker", "compose", "--project-name", "highload-lab3", "--env-file", ".env", "-f", "compose.yml"] def inspect(container): # Deliberately inspect only public identity/state, not credentials or env. raw = subprocess.check_output(["docker", "inspect", "--format", '{{json .Config.Hostname}} {{json .State.Status}}', container], text=True) hostname, state = raw.strip().split() return json.loads(hostname), json.loads(state) def restore(): subprocess.run(COMPOSE + ["up", "-d", "--wait", "--no-deps", "api"], cwd=ROOT, check=True) def main(): args = arguments(__doc__) c = Client(args.base_url, args.ca_file, timeout=2) containers = subprocess.check_output(COMPOSE + ["ps", "-q", "api"], cwd=ROOT, text=True).split() check(len(containers) >= 2, "need at least two running API containers") targets = [(container, inspect(container)[0]) for container in containers] alice, bob, _, chat = c.bootstrap() records = [] for container, target in targets: # Read through the LB using one existing session until all replicas rejoin. seen = set() deadline = time.monotonic() + 10 while len(seen) < len(targets) and time.monotonic() < deadline: seen.add(c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"]) time.sleep(0.05) check(target in seen and len(seen) >= 2, "restored replica is not serving through HAProxy") confirmed_key = uuid.uuid4().hex confirmed_text = "Before failure " + confirmed_key original = c.message(alice, chat, confirmed_text, key=confirmed_key)["json"] env = dict(os.environ, TARGET_INSTANCE_ID=target) addresses = {network["IPAddress"] for network in identity(container)["networks"].values()} slot = next(row["svname"] for row in stats() if row["addr"].split(":")[0] in addresses) started = time.monotonic() proc = subprocess.Popen([str(ROOT / "scripts" / "stop-one.sh")], cwd=ROOT, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) successes, last_instances = [], [] write_errors = read_errors = reads = 0 first_write = first_read = None excluded = None key = uuid.uuid4().hex try: while time.monotonic() - started < 15: check(proc.poll() in (None, 0), "stop-one hook failed") if excluded is None and any(row["svname"] == slot and not row["status"].startswith("UP") for row in stats()): excluded = time.monotonic() - started try: result = c.message(alice, chat, "During failure " + key, key=key) successes.append(result["json"]) if proc.poll() == 0: last_instances.append(result["headers"]["X-Instance-Id"]) if first_write is None: first_write = time.monotonic() - started key = uuid.uuid4().hex except Failure: write_errors += 1 # Retry the SAME key and text on the next iteration. try: c.call("GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=bob, schema="MessagePage") reads += 1 if proc.poll() == 0 and first_read is None: first_read = time.monotonic() - started except Failure: read_errors += 1 time.sleep(0.1) elapsed = time.monotonic() - started check(proc.poll() == 0, "stop-one hook did not finish") check(inspect(container)[1] == "exited", "target container did not stay stopped") check(first_write is not None and first_write <= 10, "writes did not recover within 10 seconds") check(first_read is not None and first_read <= 10, "reads did not recover within 10 seconds") check(excluded is not None and excluded <= 6, "HAProxy did not exclude stopped backend within the 6-second budget") check(len(last_instances) >= 5 and target not in last_instances[-5:], "stopped replica still serves responses") check(c.message(alice, chat, confirmed_text, key=confirmed_key)["json"] == original, "replay after switching replicas changed a confirmed message") messages = history(c, bob, chat) check(all(message in messages for message in [original] + successes), "confirmed message lost") texts = [message["text"] for message in messages] check(len(texts) == len(set(texts)), "uncertain retry created a duplicate") record = {"stopped_instance": target, "container_state": "exited", "successful_writes": len(successes), "successful_reads": reads, "write_errors": write_errors, "read_errors": read_errors, "first_write_seconds": round(first_write, 3), "first_read_seconds": round(first_read, 3)} record.update(elapsed_seconds=round(elapsed, 3), lb_exclusion_seconds=round(excluded, 3), successful_rps=round((len(successes) + reads) / elapsed, 3), attempted_rps=round((len(successes) + reads + write_errors + read_errors) / elapsed, 3), workload="one sequential writer and reader, 100ms pause; not a capacity benchmark") records.append(record) print(json.dumps(record), flush=True) finally: if proc.poll() is None: proc.terminate() proc.wait(timeout=5) restore() ROOT.joinpath(".memory/grade4").mkdir(parents=True, exist_ok=True) ROOT.joinpath(".memory/grade4/failover.json").write_text(json.dumps(records, indent=2) + "\n") print("PASS: each API replica stopped in turn; reads, writes, sessions and confirmed messages preserved") if __name__ == "__main__": run(main)