Files
mtusi2026lab4/tests/replica_failover.py
2026-11-17 02:09:04 +03:00

147 lines
5.7 KiB
Python

#!/usr/bin/env python3
"""Fail one explicitly selected replica and write sanitized evidence."""
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
from urllib.parse import quote
from client import Client, Failure, arguments, check, run
def history(client, actor, chat):
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
messages = []
seen = set()
while True:
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
messages.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
return messages
check(cursor not in seen, "history cursor cycle")
seen.add(cursor)
path = (
"/api/v1/chats/"
+ chat["id"]
+ "/messages?limit=100&cursor="
+ quote(cursor, safe="")
)
def main():
def configure(parser):
parser.add_argument("--target-id", required=True)
parser.add_argument("--action-script", required=True)
parser.add_argument("--output")
args = arguments("Targeted API failover evidence", configure)
target = args.target_id
script = Path(args.action_script).expanduser().resolve()
check(script.is_file() and os.access(script, os.X_OK), "action script must be executable")
client = Client(args.base_url, args.ca_file, timeout=2)
alice, bob, _, chat = client.bootstrap()
observed_before = {
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"][
"X-Instance-Id"
]
for _ in range(40)
}
check(len(observed_before) >= 2, "fewer than two API replicas observed")
check(target in observed_before, "target ID is not a currently observed API replica")
original = None
for _ in range(20):
original = client.message(
alice, chat, "pre-failure-" + uuid.uuid4().hex, key=uuid.uuid4().hex
)
if original["headers"]["X-Instance-Id"] == target:
break
check(original["headers"]["X-Instance-Id"] == target, "could not select requested replica")
env = dict(os.environ)
env["TARGET_INSTANCE_ID"] = target
process = subprocess.Popen(
[str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
)
started = time.monotonic()
recovered_at = None
successful_writes = []
failed_writes = 0
failed_reads = 0
served_after = []
key = uuid.uuid4().hex
text = "during-failover-" + key
try:
while time.monotonic() - started < 15:
status = process.poll()
check(status is None or status == 0, "action script failed")
check(status is not None or time.monotonic() - started < 10, "action script exceeded 10s")
try:
read = client.call("GET", "/api/v1/users/me", actor=alice, schema="User")
served_after.append(read["headers"]["X-Instance-Id"])
except Failure:
failed_reads += 1
time.sleep(0.1)
continue
try:
response = client.message(alice, chat, text, key=key)
successful_writes.append(response["json"])
if status == 0 and recovered_at is None:
recovered_at = time.monotonic() - started
key = uuid.uuid4().hex
text = "during-failover-" + key
except Failure:
failed_writes += 1
time.sleep(0.1)
check(process.poll() == 0, "action script did not finish successfully")
check(recovered_at is not None and recovered_at <= 10, "writes did not recover within 10s")
check(len(successful_writes) >= 5, "too few successful writes after the failure")
check(target not in served_after[-5:], "stopped replica still serves requests")
all_messages = history(client, bob, chat)
ids = [message["id"] for message in all_messages]
texts = [message["text"] for message in all_messages]
check(len(ids) == len(set(ids)), "duplicate message IDs in history")
check(len(texts) == len(set(texts)), "retry created duplicate message bodies")
check(original["json"] in all_messages, "confirmed pre-failure message was lost")
for message in successful_writes:
check(message in all_messages, "confirmed failover write was lost")
elapsed = time.monotonic() - started
total_attempts = len(successful_writes) + failed_writes
result = {
"target_instance_id": target,
"observed_instances_before": len(observed_before),
"instances_serving_after": sorted(set(served_after)),
"duration_seconds": round(elapsed, 3),
"successful_writes": len(successful_writes),
"failed_write_attempts": failed_writes,
"failed_read_attempts": failed_reads,
"successful_rps": round(len(successful_writes) / elapsed, 3),
"write_error_ratio": round(failed_writes / total_attempts, 6),
"recovery_seconds": round(recovered_at, 3),
"confirmed_data_preserved": True,
"duplicate_messages": 0,
}
rendered = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
if args.output:
destination = Path(args.output)
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(rendered, encoding="utf-8")
print(rendered, end="")
finally:
if process.poll() is None:
process.terminate()
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
process.kill()
process.wait()
if __name__ == "__main__":
run(main)