forked from mtusi2026/mtusi2026lab3
106 lines
4.1 KiB
Python
106 lines
4.1 KiB
Python
"""Stop api-1 and api-2 in turn and record bounded failover metrics."""
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure, arguments, check, run # noqa: E402
|
|
|
|
|
|
def command(*args, env=None):
|
|
subprocess.run(args, cwd=ROOT, env=env, check=True,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
|
|
|
|
def wait_instances(client, expected, timeout=60):
|
|
deadline = time.monotonic() + timeout
|
|
found = set()
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
found.add(client.call("GET", "/health/ready", schema="Health")
|
|
["headers"]["X-Instance-Id"])
|
|
if expected.issubset(found):
|
|
return
|
|
except Failure:
|
|
pass
|
|
time.sleep(0.1)
|
|
raise Failure("expected API replicas did not become visible")
|
|
|
|
|
|
def main():
|
|
args = arguments("Sequential API failover metrics")
|
|
client = Client(args.base_url, args.ca_file, timeout=2)
|
|
command(str(ROOT / "scripts" / "start-api.sh"))
|
|
wait_instances(client, {"api-1", "api-2"})
|
|
alice, _, _, chat = client.bootstrap()
|
|
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
|
results = []
|
|
|
|
for target in ("api-1", "api-2"):
|
|
command(str(ROOT / "scripts" / "start-api.sh"))
|
|
wait_instances(client, {"api-1", "api-2"})
|
|
env = dict(os.environ)
|
|
env["TARGET_INSTANCE_ID"] = target
|
|
started = time.monotonic()
|
|
process = subprocess.Popen([str(ROOT / "scripts" / "stop-one.sh")],
|
|
cwd=ROOT, env=env,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL)
|
|
successes = failures = 0
|
|
first_survivor = None
|
|
deadline = started + 8
|
|
key = uuid.uuid4().hex
|
|
text = "failover-" + target + "-" + key
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
response = client.call("POST", path, 201, actor=alice,
|
|
body={"text": text},
|
|
headers={"Idempotency-Key": key},
|
|
schema="Message")
|
|
successes += 1
|
|
if (process.poll() is not None
|
|
and response["headers"]["X-Instance-Id"] != target
|
|
and first_survivor is None):
|
|
first_survivor = time.monotonic() - started
|
|
key = uuid.uuid4().hex
|
|
text = "failover-" + target + "-" + key
|
|
except Failure:
|
|
failures += 1
|
|
# Repeat the same body and key after an uncertain result.
|
|
code = process.wait(timeout=10)
|
|
elapsed = time.monotonic() - started
|
|
check(code == 0, "stop-one.sh failed")
|
|
check(successes > 0 and first_survivor is not None,
|
|
"surviving replica did not accept writes")
|
|
check(first_survivor <= 10, "failover exceeded ten seconds")
|
|
results.append({
|
|
"stopped_instance": target,
|
|
"duration_seconds": round(elapsed, 3),
|
|
"successful_requests": successes,
|
|
"failed_requests": failures,
|
|
"successful_rps": round(successes / elapsed, 3),
|
|
"first_survivor_success_seconds": round(first_survivor, 3),
|
|
})
|
|
|
|
command(str(ROOT / "scripts" / "start-api.sh"))
|
|
wait_instances(client, {"api-1", "api-2"})
|
|
output = {"workload": "single-client closed-loop POST messages", "runs": results}
|
|
(ROOT / "evidence").mkdir(exist_ok=True)
|
|
(ROOT / "evidence" / "failover-grade4.json").write_text(
|
|
json.dumps(output, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
print("PASS:", json.dumps(output, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
run(main)
|
|
finally:
|
|
subprocess.run([str(ROOT / "scripts" / "start-api.sh")], cwd=ROOT,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
check=False)
|