Files
mtusi2026lab3/scripts/test_readiness_routing.py
2026-10-25 21:46:21 +03:00

199 lines
7.2 KiB
Python
Executable File

#!/usr/bin/env python3
"""Prove that the gateway excludes an API replica which loses readiness."""
import argparse
from collections import Counter
import http.client
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
FAULT_IMAGE = "node:24-alpine@sha256:44fce696f38ba8c987c4db89954051000bbd05ffd8d8659ba15ed4a8602e937f"
def command(args, timeout=180, check=True):
result = subprocess.run(
[str(value) for value in args],
cwd=ROOT,
text=True,
capture_output=True,
timeout=timeout,
check=False,
)
if check and result.returncode:
detail = (result.stderr or result.stdout).strip().splitlines()
suffix = f": {detail[-1]}" if detail else ""
raise RuntimeError(f"{' '.join(map(str, args[:4]))} failed{suffix}")
return result
def compose(env_file, *args):
return command(["docker", "compose", "--env-file", env_file, *args])
def inspect(container_id):
return json.loads(command([
"docker", "inspect", "--format", "{{json .}}", container_id,
]).stdout)
def call(host, port, method, path, body=None, headers=None, timeout=5):
connection = http.client.HTTPConnection(host, port, timeout=timeout)
connection.request(method, path, body=body, headers=headers or {})
response = connection.getresponse()
payload = response.read()
result = {
"status": response.status,
"instance": response.getheader("X-Instance-Id", ""),
"body": payload,
}
connection.close()
return result
def observe_instances(base_host, base_port, requests=16):
instances = Counter()
statuses = Counter()
for _ in range(requests):
result = call(base_host, base_port, "GET", "/health/live")
statuses[result["status"]] += 1
instances[result["instance"]] += 1
return statuses, instances
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--env-file", default=".env")
parser.add_argument("--host", default="localhost")
parser.add_argument("--port", type=int, default=8080)
parser.add_argument("--output", default="")
args = parser.parse_args()
api_ids = [line for line in compose(args.env_file, "ps", "-q", "api").stdout.splitlines() if line]
if len(api_ids) != 2:
raise RuntimeError("readiness routing test requires exactly two API replicas")
target_id = api_ids[0]
target = inspect(target_id)
target_instance = target["Config"]["Hostname"]
target_ip = next(iter(target["NetworkSettings"]["Networks"].values()))["IPAddress"]
postgres_id = compose(args.env_file, "ps", "-q", "postgres").stdout.strip()
postgres = inspect(postgres_id)
postgres_ip = next(iter(postgres["NetworkSettings"]["Networks"].values()))["IPAddress"]
before_statuses, before_instances = observe_instances(args.host, args.port, 40)
if before_statuses != {200: 40} or len(before_instances) != 2:
raise RuntimeError("both API replicas must serve traffic before fault injection")
helper = f"mtusi2026lab3-readiness-fault-{os.getpid()}"
rule = [
"iptables", "-D", "OUTPUT", "-d", postgres_ip,
"-p", "tcp", "--dport", "5432", "-j", "REJECT",
"--reject-with", "tcp-reset",
]
detection_started = time.monotonic()
try:
script = (
"apk add --no-cache iptables >/dev/null && "
f"iptables -I OUTPUT -d {postgres_ip} -p tcp --dport 5432 "
"-j REJECT --reject-with tcp-reset && "
"touch /tmp/fault-ready && sleep 180"
)
command([
"docker", "run", "-d", "--name", helper,
"--network", f"container:{target_id}",
"--cap-add", "NET_ADMIN", FAULT_IMAGE,
"sh", "-c", script,
])
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
ready = command(
["docker", "exec", helper, "test", "-f", "/tmp/fault-ready"],
check=False,
)
if ready.returncode == 0:
break
time.sleep(0.2)
else:
raise RuntimeError("fault helper did not become ready")
direct = call(target_ip, 8080, "GET", "/health/ready")
if direct["status"] != 503:
raise RuntimeError(f"target readiness is {direct['status']}, want 503")
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
statuses, instances = observe_instances(args.host, args.port, 16)
if statuses == {200: 16} and target_instance not in instances:
break
time.sleep(0.25)
else:
raise RuntimeError("gateway did not exclude the unready API within 10 seconds")
detection_seconds = time.monotonic() - detection_started
health_statuses, health_instances = observe_instances(args.host, args.port, 40)
registration_statuses = Counter()
registration_instances = Counter()
for _ in range(20):
body = json.dumps({
"username": "readiness_" + uuid.uuid4().hex[:16],
"display_name": "Readiness audit",
"password": "T3st-" + uuid.uuid4().hex,
}).encode()
response = call(
args.host,
args.port,
"POST",
"/api/v1/users",
body=body,
headers={"Content-Type": "application/json"},
timeout=8,
)
registration_statuses[response["status"]] += 1
registration_instances[response["instance"]] += 1
if health_statuses != {200: 40} or target_instance in health_instances:
raise RuntimeError("unready API received liveness traffic through gateway")
if registration_statuses != {201: 20} or target_instance in registration_instances:
raise RuntimeError("unready API received registration traffic through gateway")
finally:
command(["docker", "exec", helper, *rule], check=False)
command(["docker", "rm", "-f", helper], check=False)
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
direct = call(target_ip, 8080, "GET", "/health/ready")
if direct["status"] == 200:
_, instances = observe_instances(args.host, args.port, 24)
if target_instance in instances:
break
time.sleep(0.25)
else:
raise RuntimeError("recovered API did not return to gateway rotation")
result = {
"target_instance": target_instance,
"detection_seconds": round(detection_seconds, 3),
"health_requests": dict(health_statuses),
"registration_requests": dict(registration_statuses),
"recovered": True,
}
rendered = json.dumps(result, ensure_ascii=False, indent=2)
if args.output:
output = Path(args.output)
if not output.is_absolute():
output = ROOT / output
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(rendered + "\n", encoding="utf-8")
print(rendered)
print("PASS: readiness-aware gateway excluded and restored one API replica")
if __name__ == "__main__":
main()