121 lines
4.3 KiB
Python
121 lines
4.3 KiB
Python
"""Stop each real API process under traffic; retain confirmed writes and source aliases."""
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure
|
|
|
|
|
|
def compose(*args):
|
|
return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True)
|
|
|
|
|
|
def aliases():
|
|
result = {}
|
|
for cid in compose("ps", "-q", "api").split():
|
|
alias = subprocess.check_output(
|
|
[
|
|
"docker",
|
|
"exec",
|
|
cid,
|
|
"python",
|
|
"-c",
|
|
"import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
|
|
],
|
|
text=True,
|
|
).strip()
|
|
result[cid] = alias
|
|
return result
|
|
|
|
|
|
def main():
|
|
compose("up", "-d", "--no-recreate", "--wait")
|
|
subprocess.run([sys.executable, "scripts/wait_replicas.py", "2"], cwd=ROOT, check=True)
|
|
c = Client("http://127.0.0.1:8083")
|
|
actor, _, _, chat = c.bootstrap()
|
|
initial = aliases()
|
|
runs = []
|
|
for target_container, target_alias in initial.items():
|
|
successful, errors = [], []
|
|
stop = threading.Event()
|
|
stopped_at = None
|
|
|
|
def traffic():
|
|
while not stop.is_set():
|
|
try:
|
|
response = c.message(actor, chat, "failover-proof")
|
|
successful.append(
|
|
{
|
|
"at": time.monotonic(),
|
|
"id": response["json"]["id"],
|
|
"instance": response["headers"]["X-Instance-Id"],
|
|
}
|
|
)
|
|
except Failure:
|
|
errors.append(time.monotonic())
|
|
|
|
started = time.monotonic()
|
|
thread = threading.Thread(target=traffic)
|
|
thread.start()
|
|
time.sleep(1)
|
|
stopped_at = time.monotonic()
|
|
subprocess.run(
|
|
["scripts/stop-one.sh"],
|
|
cwd=ROOT,
|
|
env={**os.environ, "TARGET_INSTANCE_ID": target_alias},
|
|
check=True,
|
|
)
|
|
time.sleep(4)
|
|
stop.set()
|
|
thread.join()
|
|
elapsed = time.monotonic() - started
|
|
after = [x for x in successful if x["at"] > stopped_at]
|
|
assert after and all(x["instance"] != target_alias for x in after[-5:])
|
|
assert min(x["at"] for x in after) - stopped_at <= 10
|
|
state = json.loads(
|
|
subprocess.check_output(["docker", "inspect", target_container], text=True)
|
|
)[0]["State"]
|
|
assert not state["Running"]
|
|
runs.append(
|
|
{
|
|
"target_container": target_container,
|
|
"target_alias": target_alias,
|
|
"successful_writes": len(successful),
|
|
"failed_attempts": len(errors),
|
|
"successful_rps": len(successful) / elapsed,
|
|
"recovery_seconds": min(x["at"] for x in after) - stopped_at,
|
|
"observed_after": sorted({x["instance"] for x in after[-5:]}),
|
|
"target_running_after_hook": state["Running"],
|
|
}
|
|
)
|
|
subprocess.run(["docker", "start", target_container], check=True, stdout=subprocess.DEVNULL)
|
|
subprocess.run([sys.executable, "scripts/wait_replicas.py", "2"], cwd=ROOT, check=True)
|
|
# Verify every acknowledged write, without assuming failed requests did not commit.
|
|
items, cursor = [], None
|
|
while True:
|
|
path = f"/api/v1/chats/{chat['id']}/messages?limit=100"
|
|
if cursor:
|
|
path += "&cursor=" + cursor
|
|
page = c.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
|
items.extend(page["items"])
|
|
cursor = page["next_cursor"]
|
|
if cursor is None:
|
|
break
|
|
ids = {x["id"] for x in items}
|
|
assert all(x["id"] in ids for x in successful)
|
|
runs[-1]["all_confirmed_present"] = True
|
|
report = {"real_instances_before": initial, "runs": runs, "pass": True}
|
|
(ROOT / "evidence/replica-failures.json").write_text(json.dumps(report, indent=2) + "\n")
|
|
print(json.dumps(report, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|