forked from mtusi2026/mtusi2026lab3
test: cover persistence failover and graceful draining
Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
This commit is contained in:
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check failover of a named Compose API replica without changing public tests."""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--target-index", type=int, choices=(1, 2), required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
ids = subprocess.run(
|
||||
["docker", "compose", "ps", "--all", "-q", "api"],
|
||||
cwd=ROOT, capture_output=True, text=True, check=True,
|
||||
).stdout.splitlines()
|
||||
check(len(ids) == 2, "targeted failover requires two API replicas")
|
||||
target = subprocess.run(
|
||||
["docker", "inspect", "--format", "{{.Config.Hostname}}",
|
||||
ids[args.target_index - 1]],
|
||||
cwd=ROOT, capture_output=True, text=True, check=True,
|
||||
).stdout.strip()
|
||||
expected = {
|
||||
subprocess.run(
|
||||
["docker", "inspect", "--format", "{{.Config.Hostname}}", item],
|
||||
cwd=ROOT, capture_output=True, text=True, check=True,
|
||||
).stdout.strip()
|
||||
for item in ids
|
||||
}
|
||||
|
||||
client = Client(args.base_url, timeout=10)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
before = client.message(alice, chat, "Before targeted failover")["json"]
|
||||
observed = {
|
||||
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")[
|
||||
"headers"
|
||||
]["X-Instance-Id"]
|
||||
for _ in range(40)
|
||||
}
|
||||
check(observed == expected, "both replicas must answer before failure")
|
||||
client.timeout = 2
|
||||
|
||||
hook = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh")
|
||||
env = dict(os.environ, TARGET_INSTANCE_ID=target)
|
||||
started = time.monotonic()
|
||||
result = subprocess.run(
|
||||
[str(hook)], cwd=ROOT, env=env, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL, timeout=10, check=False,
|
||||
)
|
||||
check(result.returncode == 0, "stop-one hook failed")
|
||||
|
||||
key = uuid.uuid4().hex
|
||||
text = "Targeted failover " + key
|
||||
failures = 0
|
||||
while True:
|
||||
try:
|
||||
written = client.message(alice, chat, text, key=key)
|
||||
break
|
||||
except Failure:
|
||||
failures += 1
|
||||
check(time.monotonic() - started <= 10, "write recovery exceeded 10 seconds")
|
||||
time.sleep(0.1)
|
||||
recovery = time.monotonic() - started
|
||||
check(recovery <= 10, "write recovery exceeded 10 seconds")
|
||||
check(written["headers"]["X-Instance-Id"] != target,
|
||||
"stopped replica answered the write")
|
||||
|
||||
reads = 0
|
||||
first_read = None
|
||||
while reads < 10:
|
||||
try:
|
||||
response = client.call("GET", "/api/v1/users/me", actor=alice,
|
||||
schema="User")
|
||||
except Failure:
|
||||
failures += 1
|
||||
check(time.monotonic() - started <= 10,
|
||||
"read recovery exceeded 10 seconds")
|
||||
continue
|
||||
if first_read is None:
|
||||
first_read = time.monotonic() - started
|
||||
check(first_read <= 10, "read recovery exceeded 10 seconds")
|
||||
check(response["headers"]["X-Instance-Id"] != target,
|
||||
"stopped replica still serves traffic")
|
||||
reads += 1
|
||||
messages = client.call(
|
||||
"GET", f"/api/v1/chats/{chat['id']}/messages?limit=100",
|
||||
actor=bob, schema="MessagePage",
|
||||
)["json"]["items"]
|
||||
check(before in messages, "confirmed pre-failure message disappeared")
|
||||
check(sum(message["text"] == text for message in messages) == 1,
|
||||
"uncertain retry created a duplicate")
|
||||
print(f"PASS: replica {args.target_index} stopped; write_recovery={recovery:.3f}s; "
|
||||
f"read_recovery={first_read:.3f}s; transient_failures={failures}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user