forked from mtusi2026/mtusi2026lab7
111 lines
3.8 KiB
Python
111 lines
3.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Stop each exact API container under writes, preserving sessions and original files."""
|
|
|
|
# ruff: noqa: E402 -- repository scripts are imported after setting their path.
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
from client import Client, Failure
|
|
from replicas import compose, containers, describe, instance_id, wait_ready
|
|
from resilience import history
|
|
|
|
|
|
def main():
|
|
c = Client("https://localhost:8443", str(ROOT / ".local/trust/ca.crt"), timeout=2)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
attachment, content = c.attachment(alice, chat)
|
|
c.wait_ready(alice, attachment)
|
|
expected = [c.message(alice, chat, "Before failures")["json"]]
|
|
targets = containers()
|
|
assert len(targets) == 2
|
|
results = []
|
|
try:
|
|
for container in targets:
|
|
target = instance_id(container)
|
|
env = {**os.environ, "TARGET_INSTANCE_ID": target}
|
|
started = time.monotonic()
|
|
process = subprocess.Popen(
|
|
[str(ROOT / "scripts/stop-one.sh")],
|
|
cwd=ROOT,
|
|
env=env,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
first = None
|
|
failures = 0
|
|
served = []
|
|
success = 0
|
|
key = uuid.uuid4().hex
|
|
text = "During failure " + key
|
|
while time.monotonic() - started < 15:
|
|
assert process.poll() in (None, 0)
|
|
try:
|
|
response = c.message(alice, chat, text, key=key)
|
|
expected.append(response["json"])
|
|
success += 1
|
|
if process.poll() == 0:
|
|
if first is None:
|
|
first = time.monotonic() - started
|
|
served.append(response["headers"]["X-Instance-Id"])
|
|
key = uuid.uuid4().hex
|
|
text = "During failure " + key
|
|
except Failure:
|
|
failures += 1
|
|
time.sleep(0.15)
|
|
assert process.wait(timeout=1) == 0
|
|
assert first is not None and first <= 10
|
|
assert len(served) >= 5 and target not in served[-5:]
|
|
assert not describe(container)["Running"]
|
|
messages = history(c, bob, chat)
|
|
assert len({x["id"] for x in messages}) == len(messages)
|
|
assert len({x["text"] for x in messages}) == len(messages)
|
|
assert all(x in messages for x in expected)
|
|
assert (
|
|
c.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob)[
|
|
"bytes"
|
|
]
|
|
== content
|
|
)
|
|
results.append(
|
|
{
|
|
"container": container[:12],
|
|
"target_instance": target,
|
|
"successful_writes": success,
|
|
"failed_attempts": failures,
|
|
"first_recovery_seconds": round(first, 3),
|
|
"result": "PASS",
|
|
}
|
|
)
|
|
compose("start", "api")
|
|
wait_ready(container)
|
|
time.sleep(2)
|
|
finally:
|
|
compose("start", "api")
|
|
(ROOT / "evidence/failover-summary.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"result": "PASS",
|
|
"distinct_containers": len(results),
|
|
"runs": len(results),
|
|
"events": results,
|
|
},
|
|
indent=2,
|
|
)
|
|
+ "\n"
|
|
)
|
|
print(
|
|
"PASS: both exact API containers stopped under writes; sessions, messages and attachment preserved"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|