#!/usr/bin/env python3 """Prove that the gateway excludes an API replica which loses readiness.""" import argparse from collections import Counter import http.client import json import os from pathlib import Path import subprocess import time import uuid ROOT = Path(__file__).resolve().parents[1] FAULT_IMAGE = "node:24-alpine@sha256:44fce696f38ba8c987c4db89954051000bbd05ffd8d8659ba15ed4a8602e937f" def command(args, timeout=180, check=True): result = subprocess.run( [str(value) for value in args], cwd=ROOT, text=True, capture_output=True, timeout=timeout, check=False, ) if check and result.returncode: detail = (result.stderr or result.stdout).strip().splitlines() suffix = f": {detail[-1]}" if detail else "" raise RuntimeError(f"{' '.join(map(str, args[:4]))} failed{suffix}") return result def compose(env_file, *args): return command(["docker", "compose", "--env-file", env_file, *args]) def inspect(container_id): return json.loads(command([ "docker", "inspect", "--format", "{{json .}}", container_id, ]).stdout) def call(host, port, method, path, body=None, headers=None, timeout=5): connection = http.client.HTTPConnection(host, port, timeout=timeout) connection.request(method, path, body=body, headers=headers or {}) response = connection.getresponse() payload = response.read() result = { "status": response.status, "instance": response.getheader("X-Instance-Id", ""), "body": payload, } connection.close() return result def observe_instances(base_host, base_port, requests=16): instances = Counter() statuses = Counter() for _ in range(requests): result = call(base_host, base_port, "GET", "/health/live") statuses[result["status"]] += 1 instances[result["instance"]] += 1 return statuses, instances def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--env-file", default=".env") parser.add_argument("--host", default="localhost") parser.add_argument("--port", type=int, default=8080) parser.add_argument("--output", default="") args = parser.parse_args() api_ids = [line for line in compose(args.env_file, "ps", "-q", "api").stdout.splitlines() if line] if len(api_ids) != 2: raise RuntimeError("readiness routing test requires exactly two API replicas") target_id = api_ids[0] target = inspect(target_id) target_instance = target["Config"]["Hostname"] target_ip = next(iter(target["NetworkSettings"]["Networks"].values()))["IPAddress"] postgres_id = compose(args.env_file, "ps", "-q", "postgres").stdout.strip() postgres = inspect(postgres_id) postgres_ip = next(iter(postgres["NetworkSettings"]["Networks"].values()))["IPAddress"] before_statuses, before_instances = observe_instances(args.host, args.port, 40) if before_statuses != {200: 40} or len(before_instances) != 2: raise RuntimeError("both API replicas must serve traffic before fault injection") helper = f"mtusi2026lab3-readiness-fault-{os.getpid()}" rule = [ "iptables", "-D", "OUTPUT", "-d", postgres_ip, "-p", "tcp", "--dport", "5432", "-j", "REJECT", "--reject-with", "tcp-reset", ] detection_started = time.monotonic() try: script = ( "apk add --no-cache iptables >/dev/null && " f"iptables -I OUTPUT -d {postgres_ip} -p tcp --dport 5432 " "-j REJECT --reject-with tcp-reset && " "touch /tmp/fault-ready && sleep 180" ) command([ "docker", "run", "-d", "--name", helper, "--network", f"container:{target_id}", "--cap-add", "NET_ADMIN", FAULT_IMAGE, "sh", "-c", script, ]) deadline = time.monotonic() + 30 while time.monotonic() < deadline: ready = command( ["docker", "exec", helper, "test", "-f", "/tmp/fault-ready"], check=False, ) if ready.returncode == 0: break time.sleep(0.2) else: raise RuntimeError("fault helper did not become ready") direct = call(target_ip, 8080, "GET", "/health/ready") if direct["status"] != 503: raise RuntimeError(f"target readiness is {direct['status']}, want 503") deadline = time.monotonic() + 10 while time.monotonic() < deadline: statuses, instances = observe_instances(args.host, args.port, 16) if statuses == {200: 16} and target_instance not in instances: break time.sleep(0.25) else: raise RuntimeError("gateway did not exclude the unready API within 10 seconds") detection_seconds = time.monotonic() - detection_started health_statuses, health_instances = observe_instances(args.host, args.port, 40) registration_statuses = Counter() registration_instances = Counter() for _ in range(20): body = json.dumps({ "username": "readiness_" + uuid.uuid4().hex[:16], "display_name": "Readiness audit", "password": "T3st-" + uuid.uuid4().hex, }).encode() response = call( args.host, args.port, "POST", "/api/v1/users", body=body, headers={"Content-Type": "application/json"}, timeout=8, ) registration_statuses[response["status"]] += 1 registration_instances[response["instance"]] += 1 if health_statuses != {200: 40} or target_instance in health_instances: raise RuntimeError("unready API received liveness traffic through gateway") if registration_statuses != {201: 20} or target_instance in registration_instances: raise RuntimeError("unready API received registration traffic through gateway") finally: command(["docker", "exec", helper, *rule], check=False) command(["docker", "rm", "-f", helper], check=False) deadline = time.monotonic() + 15 while time.monotonic() < deadline: direct = call(target_ip, 8080, "GET", "/health/ready") if direct["status"] == 200: _, instances = observe_instances(args.host, args.port, 24) if target_instance in instances: break time.sleep(0.25) else: raise RuntimeError("recovered API did not return to gateway rotation") result = { "target_instance": target_instance, "detection_seconds": round(detection_seconds, 3), "health_requests": dict(health_statuses), "registration_requests": dict(registration_statuses), "recovered": True, } rendered = json.dumps(result, ensure_ascii=False, indent=2) if args.output: output = Path(args.output) if not output.is_absolute(): output = ROOT / output output.parent.mkdir(parents=True, exist_ok=True) output.write_text(rendered + "\n", encoding="utf-8") print(rendered) print("PASS: readiness-aware gateway excluded and restored one API replica") if __name__ == "__main__": main()