forked from mtusi2026/mtusi2026lab3
199 lines
7.2 KiB
Python
Executable File
199 lines
7.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Prove that the gateway excludes an API replica which loses readiness."""
|
|
|
|
import argparse
|
|
from collections import Counter
|
|
import http.client
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
FAULT_IMAGE = "node:24-alpine@sha256:44fce696f38ba8c987c4db89954051000bbd05ffd8d8659ba15ed4a8602e937f"
|
|
|
|
|
|
def command(args, timeout=180, check=True):
|
|
result = subprocess.run(
|
|
[str(value) for value in args],
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=timeout,
|
|
check=False,
|
|
)
|
|
if check and result.returncode:
|
|
detail = (result.stderr or result.stdout).strip().splitlines()
|
|
suffix = f": {detail[-1]}" if detail else ""
|
|
raise RuntimeError(f"{' '.join(map(str, args[:4]))} failed{suffix}")
|
|
return result
|
|
|
|
|
|
def compose(env_file, *args):
|
|
return command(["docker", "compose", "--env-file", env_file, *args])
|
|
|
|
|
|
def inspect(container_id):
|
|
return json.loads(command([
|
|
"docker", "inspect", "--format", "{{json .}}", container_id,
|
|
]).stdout)
|
|
|
|
|
|
def call(host, port, method, path, body=None, headers=None, timeout=5):
|
|
connection = http.client.HTTPConnection(host, port, timeout=timeout)
|
|
connection.request(method, path, body=body, headers=headers or {})
|
|
response = connection.getresponse()
|
|
payload = response.read()
|
|
result = {
|
|
"status": response.status,
|
|
"instance": response.getheader("X-Instance-Id", ""),
|
|
"body": payload,
|
|
}
|
|
connection.close()
|
|
return result
|
|
|
|
|
|
def observe_instances(base_host, base_port, requests=16):
|
|
instances = Counter()
|
|
statuses = Counter()
|
|
for _ in range(requests):
|
|
result = call(base_host, base_port, "GET", "/health/live")
|
|
statuses[result["status"]] += 1
|
|
instances[result["instance"]] += 1
|
|
return statuses, instances
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--env-file", default=".env")
|
|
parser.add_argument("--host", default="localhost")
|
|
parser.add_argument("--port", type=int, default=8080)
|
|
parser.add_argument("--output", default="")
|
|
args = parser.parse_args()
|
|
|
|
api_ids = [line for line in compose(args.env_file, "ps", "-q", "api").stdout.splitlines() if line]
|
|
if len(api_ids) != 2:
|
|
raise RuntimeError("readiness routing test requires exactly two API replicas")
|
|
target_id = api_ids[0]
|
|
target = inspect(target_id)
|
|
target_instance = target["Config"]["Hostname"]
|
|
target_ip = next(iter(target["NetworkSettings"]["Networks"].values()))["IPAddress"]
|
|
|
|
postgres_id = compose(args.env_file, "ps", "-q", "postgres").stdout.strip()
|
|
postgres = inspect(postgres_id)
|
|
postgres_ip = next(iter(postgres["NetworkSettings"]["Networks"].values()))["IPAddress"]
|
|
|
|
before_statuses, before_instances = observe_instances(args.host, args.port, 40)
|
|
if before_statuses != {200: 40} or len(before_instances) != 2:
|
|
raise RuntimeError("both API replicas must serve traffic before fault injection")
|
|
|
|
helper = f"mtusi2026lab3-readiness-fault-{os.getpid()}"
|
|
rule = [
|
|
"iptables", "-D", "OUTPUT", "-d", postgres_ip,
|
|
"-p", "tcp", "--dport", "5432", "-j", "REJECT",
|
|
"--reject-with", "tcp-reset",
|
|
]
|
|
detection_started = time.monotonic()
|
|
try:
|
|
script = (
|
|
"apk add --no-cache iptables >/dev/null && "
|
|
f"iptables -I OUTPUT -d {postgres_ip} -p tcp --dport 5432 "
|
|
"-j REJECT --reject-with tcp-reset && "
|
|
"touch /tmp/fault-ready && sleep 180"
|
|
)
|
|
command([
|
|
"docker", "run", "-d", "--name", helper,
|
|
"--network", f"container:{target_id}",
|
|
"--cap-add", "NET_ADMIN", FAULT_IMAGE,
|
|
"sh", "-c", script,
|
|
])
|
|
deadline = time.monotonic() + 30
|
|
while time.monotonic() < deadline:
|
|
ready = command(
|
|
["docker", "exec", helper, "test", "-f", "/tmp/fault-ready"],
|
|
check=False,
|
|
)
|
|
if ready.returncode == 0:
|
|
break
|
|
time.sleep(0.2)
|
|
else:
|
|
raise RuntimeError("fault helper did not become ready")
|
|
|
|
direct = call(target_ip, 8080, "GET", "/health/ready")
|
|
if direct["status"] != 503:
|
|
raise RuntimeError(f"target readiness is {direct['status']}, want 503")
|
|
|
|
deadline = time.monotonic() + 10
|
|
while time.monotonic() < deadline:
|
|
statuses, instances = observe_instances(args.host, args.port, 16)
|
|
if statuses == {200: 16} and target_instance not in instances:
|
|
break
|
|
time.sleep(0.25)
|
|
else:
|
|
raise RuntimeError("gateway did not exclude the unready API within 10 seconds")
|
|
detection_seconds = time.monotonic() - detection_started
|
|
|
|
health_statuses, health_instances = observe_instances(args.host, args.port, 40)
|
|
registration_statuses = Counter()
|
|
registration_instances = Counter()
|
|
for _ in range(20):
|
|
body = json.dumps({
|
|
"username": "readiness_" + uuid.uuid4().hex[:16],
|
|
"display_name": "Readiness audit",
|
|
"password": "T3st-" + uuid.uuid4().hex,
|
|
}).encode()
|
|
response = call(
|
|
args.host,
|
|
args.port,
|
|
"POST",
|
|
"/api/v1/users",
|
|
body=body,
|
|
headers={"Content-Type": "application/json"},
|
|
timeout=8,
|
|
)
|
|
registration_statuses[response["status"]] += 1
|
|
registration_instances[response["instance"]] += 1
|
|
|
|
if health_statuses != {200: 40} or target_instance in health_instances:
|
|
raise RuntimeError("unready API received liveness traffic through gateway")
|
|
if registration_statuses != {201: 20} or target_instance in registration_instances:
|
|
raise RuntimeError("unready API received registration traffic through gateway")
|
|
finally:
|
|
command(["docker", "exec", helper, *rule], check=False)
|
|
command(["docker", "rm", "-f", helper], check=False)
|
|
|
|
deadline = time.monotonic() + 15
|
|
while time.monotonic() < deadline:
|
|
direct = call(target_ip, 8080, "GET", "/health/ready")
|
|
if direct["status"] == 200:
|
|
_, instances = observe_instances(args.host, args.port, 24)
|
|
if target_instance in instances:
|
|
break
|
|
time.sleep(0.25)
|
|
else:
|
|
raise RuntimeError("recovered API did not return to gateway rotation")
|
|
|
|
result = {
|
|
"target_instance": target_instance,
|
|
"detection_seconds": round(detection_seconds, 3),
|
|
"health_requests": dict(health_statuses),
|
|
"registration_requests": dict(registration_statuses),
|
|
"recovered": True,
|
|
}
|
|
rendered = json.dumps(result, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
output = Path(args.output)
|
|
if not output.is_absolute():
|
|
output = ROOT / output
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
output.write_text(rendered + "\n", encoding="utf-8")
|
|
print(rendered)
|
|
print("PASS: readiness-aware gateway excluded and restored one API replica")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|