Files
mtusi2026lab3/student-tests/failover_experiment.py
T

106 lines
4.1 KiB
Python

"""Stop api-1 and api-2 in turn and record bounded failover metrics."""
import json
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
def command(*args, env=None):
subprocess.run(args, cwd=ROOT, env=env, check=True,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def wait_instances(client, expected, timeout=60):
deadline = time.monotonic() + timeout
found = set()
while time.monotonic() < deadline:
try:
found.add(client.call("GET", "/health/ready", schema="Health")
["headers"]["X-Instance-Id"])
if expected.issubset(found):
return
except Failure:
pass
time.sleep(0.1)
raise Failure("expected API replicas did not become visible")
def main():
args = arguments("Sequential API failover metrics")
client = Client(args.base_url, args.ca_file, timeout=2)
command(str(ROOT / "scripts" / "start-api.sh"))
wait_instances(client, {"api-1", "api-2"})
alice, _, _, chat = client.bootstrap()
path = "/api/v1/chats/" + chat["id"] + "/messages"
results = []
for target in ("api-1", "api-2"):
command(str(ROOT / "scripts" / "start-api.sh"))
wait_instances(client, {"api-1", "api-2"})
env = dict(os.environ)
env["TARGET_INSTANCE_ID"] = target
started = time.monotonic()
process = subprocess.Popen([str(ROOT / "scripts" / "stop-one.sh")],
cwd=ROOT, env=env,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
successes = failures = 0
first_survivor = None
deadline = started + 8
key = uuid.uuid4().hex
text = "failover-" + target + "-" + key
while time.monotonic() < deadline:
try:
response = client.call("POST", path, 201, actor=alice,
body={"text": text},
headers={"Idempotency-Key": key},
schema="Message")
successes += 1
if (process.poll() is not None
and response["headers"]["X-Instance-Id"] != target
and first_survivor is None):
first_survivor = time.monotonic() - started
key = uuid.uuid4().hex
text = "failover-" + target + "-" + key
except Failure:
failures += 1
# Repeat the same body and key after an uncertain result.
code = process.wait(timeout=10)
elapsed = time.monotonic() - started
check(code == 0, "stop-one.sh failed")
check(successes > 0 and first_survivor is not None,
"surviving replica did not accept writes")
check(first_survivor <= 10, "failover exceeded ten seconds")
results.append({
"stopped_instance": target,
"duration_seconds": round(elapsed, 3),
"successful_requests": successes,
"failed_requests": failures,
"successful_rps": round(successes / elapsed, 3),
"first_survivor_success_seconds": round(first_survivor, 3),
})
command(str(ROOT / "scripts" / "start-api.sh"))
wait_instances(client, {"api-1", "api-2"})
output = {"workload": "single-client closed-loop POST messages", "runs": results}
(ROOT / "evidence").mkdir(exist_ok=True)
(ROOT / "evidence" / "failover-grade4.json").write_text(
json.dumps(output, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print("PASS:", json.dumps(output, ensure_ascii=False))
if __name__ == "__main__":
try:
run(main)
finally:
subprocess.run([str(ROOT / "scripts" / "start-api.sh")], cwd=ROOT,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
check=False)