#!/usr/bin/env python3 """Grade-4 evidence: stop each observed API replica in turn; restore between steps.""" from __future__ import annotations import base64 import json import os import subprocess import time import uuid import urllib.error import urllib.request from pathlib import Path ROOT = Path(__file__).resolve().parents[1] BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") def call(method, path, body=None, token=None, headers=None, expected=None): h = {"Accept": "application/json"} if token: h["Authorization"] = "Bearer " + token data = None if body is not None: data = json.dumps(body).encode("utf-8") h["Content-Type"] = "application/json" if headers: h.update(headers) req = urllib.request.Request(BASE + path, data=data, headers=h, method=method) try: resp = urllib.request.urlopen(req, timeout=5) except urllib.error.HTTPError as err: resp = err raw = resp.read() status = resp.code if expected is not None and status != expected: raise RuntimeError(f"{method} {path}: expected {expected}, got {status}: {raw[:200]!r}") js = None if raw and resp.headers.get_content_type() == "application/json": js = json.loads(raw.decode("utf-8")) return {"status": status, "headers": resp.headers, "json": js} def header(resp, name: str) -> str: return resp["headers"].get(name) or resp["headers"].get(name.lower()) or "" def restore_api(target: str): replicas = os.environ.get("API_REPLICAS", "2") subprocess.check_call( [ "docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--scale", f"api={replicas}", "--remove-orphans", "api", ], cwd=ROOT, ) # Public ready is 200 via the surviving replica; wait until the LB routes to the restored one. for _ in range(180): try: if header(call("GET", "/health/ready", expected=200), "X-Instance-Id") == target: return except Exception: pass time.sleep(0.5) raise RuntimeError(f"replica {target} not back in LB rotation after restore") def one_round(round_no: int) -> dict: uname = "seq_" + uuid.uuid4().hex[:10] pw = "T3st-" + uuid.uuid4().hex user = call( "POST", "/api/v1/users", {"username": uname, "display_name": "Seq", "password": pw}, expected=201, )["json"] cred = base64.b64encode(f"{uname}:{pw}".encode()).decode() tok = call( "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}, expected=201, )["json"]["token"] peer_u = "seqb_" + uuid.uuid4().hex[:10] peer = call( "POST", "/api/v1/users", {"username": peer_u, "display_name": "SeqB", "password": pw}, expected=201, )["json"] chat = call( "POST", "/api/v1/chats", {"title": "seq", "member_ids": [user["id"], peer["id"]]}, token=tok, expected=201, )["json"] observed = set() for _ in range(40): observed.add(header(call("GET", "/api/v1/users/me", token=tok, expected=200), "X-Instance-Id")) if len(observed) < 2: raise RuntimeError(f"need ≥2 instances, got {observed}") target = sorted(observed)[round_no % len(observed)] env = dict(os.environ) env["TARGET_INSTANCE_ID"] = target script = ROOT / ("scripts/stop-one.cmd" if os.name == "nt" else "scripts/stop-one.sh") started = time.monotonic() proc = subprocess.Popen([str(script)], env=env, cwd=ROOT) ok = fail = 0 first = None after = [] while time.monotonic() - started < 20: code = proc.poll() try: key = uuid.uuid4().hex r = call( "POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "seq-" + key}, token=tok, headers={"Idempotency-Key": key}, expected=201, ) ok += 1 if code == 0: if first is None: first = time.monotonic() - started after.append(header(r, "X-Instance-Id")) except Exception: fail += 1 time.sleep(0.1) if proc.poll() is None: proc.wait(timeout=10) result = { "round": round_no, "observed_before": sorted(observed), "target": target, "successful_writes": ok, "failed_attempts": fail, "first_recovery_seconds": round(first, 3) if first is not None else None, "target_absent_in_last5": target not in after[-5:], } t0 = time.monotonic() restore_api(target) result["restore_to_lb_seconds"] = round(time.monotonic() - t0, 3) return result def main(): results = [] for i in range(2): results.append(one_round(i)) print(json.dumps(results[-1], ensure_ascii=False)) out = ROOT / "evidence" / "seq-failover.json" out.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8") print("WROTE", out) if __name__ == "__main__": main()