forked from mtusi2026/mtusi2026lab3
147 lines
5.7 KiB
Python
147 lines
5.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Fail one explicitly selected replica and write sanitized evidence."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
from urllib.parse import quote
|
|
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
|
|
def history(client, actor, chat):
|
|
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
|
|
messages = []
|
|
seen = set()
|
|
while True:
|
|
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
|
messages.extend(page["items"])
|
|
cursor = page["next_cursor"]
|
|
if cursor is None:
|
|
return messages
|
|
check(cursor not in seen, "history cursor cycle")
|
|
seen.add(cursor)
|
|
path = (
|
|
"/api/v1/chats/"
|
|
+ chat["id"]
|
|
+ "/messages?limit=100&cursor="
|
|
+ quote(cursor, safe="")
|
|
)
|
|
|
|
|
|
def main():
|
|
def configure(parser):
|
|
parser.add_argument("--target-id", required=True)
|
|
parser.add_argument("--action-script", required=True)
|
|
parser.add_argument("--output")
|
|
|
|
args = arguments("Targeted API failover evidence", configure)
|
|
target = args.target_id
|
|
script = Path(args.action_script).expanduser().resolve()
|
|
check(script.is_file() and os.access(script, os.X_OK), "action script must be executable")
|
|
client = Client(args.base_url, args.ca_file, timeout=2)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
observed_before = {
|
|
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"][
|
|
"X-Instance-Id"
|
|
]
|
|
for _ in range(40)
|
|
}
|
|
check(len(observed_before) >= 2, "fewer than two API replicas observed")
|
|
check(target in observed_before, "target ID is not a currently observed API replica")
|
|
|
|
original = None
|
|
for _ in range(20):
|
|
original = client.message(
|
|
alice, chat, "pre-failure-" + uuid.uuid4().hex, key=uuid.uuid4().hex
|
|
)
|
|
if original["headers"]["X-Instance-Id"] == target:
|
|
break
|
|
check(original["headers"]["X-Instance-Id"] == target, "could not select requested replica")
|
|
|
|
env = dict(os.environ)
|
|
env["TARGET_INSTANCE_ID"] = target
|
|
process = subprocess.Popen(
|
|
[str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
|
)
|
|
started = time.monotonic()
|
|
recovered_at = None
|
|
successful_writes = []
|
|
failed_writes = 0
|
|
failed_reads = 0
|
|
served_after = []
|
|
key = uuid.uuid4().hex
|
|
text = "during-failover-" + key
|
|
try:
|
|
while time.monotonic() - started < 15:
|
|
status = process.poll()
|
|
check(status is None or status == 0, "action script failed")
|
|
check(status is not None or time.monotonic() - started < 10, "action script exceeded 10s")
|
|
try:
|
|
read = client.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
|
served_after.append(read["headers"]["X-Instance-Id"])
|
|
except Failure:
|
|
failed_reads += 1
|
|
time.sleep(0.1)
|
|
continue
|
|
try:
|
|
response = client.message(alice, chat, text, key=key)
|
|
successful_writes.append(response["json"])
|
|
if status == 0 and recovered_at is None:
|
|
recovered_at = time.monotonic() - started
|
|
key = uuid.uuid4().hex
|
|
text = "during-failover-" + key
|
|
except Failure:
|
|
failed_writes += 1
|
|
time.sleep(0.1)
|
|
|
|
check(process.poll() == 0, "action script did not finish successfully")
|
|
check(recovered_at is not None and recovered_at <= 10, "writes did not recover within 10s")
|
|
check(len(successful_writes) >= 5, "too few successful writes after the failure")
|
|
check(target not in served_after[-5:], "stopped replica still serves requests")
|
|
all_messages = history(client, bob, chat)
|
|
ids = [message["id"] for message in all_messages]
|
|
texts = [message["text"] for message in all_messages]
|
|
check(len(ids) == len(set(ids)), "duplicate message IDs in history")
|
|
check(len(texts) == len(set(texts)), "retry created duplicate message bodies")
|
|
check(original["json"] in all_messages, "confirmed pre-failure message was lost")
|
|
for message in successful_writes:
|
|
check(message in all_messages, "confirmed failover write was lost")
|
|
|
|
elapsed = time.monotonic() - started
|
|
total_attempts = len(successful_writes) + failed_writes
|
|
result = {
|
|
"target_instance_id": target,
|
|
"observed_instances_before": len(observed_before),
|
|
"instances_serving_after": sorted(set(served_after)),
|
|
"duration_seconds": round(elapsed, 3),
|
|
"successful_writes": len(successful_writes),
|
|
"failed_write_attempts": failed_writes,
|
|
"failed_read_attempts": failed_reads,
|
|
"successful_rps": round(len(successful_writes) / elapsed, 3),
|
|
"write_error_ratio": round(failed_writes / total_attempts, 6),
|
|
"recovery_seconds": round(recovered_at, 3),
|
|
"confirmed_data_preserved": True,
|
|
"duplicate_messages": 0,
|
|
}
|
|
rendered = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
|
if args.output:
|
|
destination = Path(args.output)
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_text(rendered, encoding="utf-8")
|
|
print(rendered, end="")
|
|
finally:
|
|
if process.poll() is None:
|
|
process.terminate()
|
|
try:
|
|
process.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
process.kill()
|
|
process.wait()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|