Files
2026-11-20 15:30:33 +03:00

111 lines
3.8 KiB
Python

#!/usr/bin/env python3
"""Stop each exact API container under writes, preserving sessions and original files."""
# ruff: noqa: E402 -- repository scripts are imported after setting their path.
import json
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from client import Client, Failure
from replicas import compose, containers, describe, instance_id, wait_ready
from resilience import history
def main():
c = Client("https://localhost:8443", str(ROOT / ".local/trust/ca.crt"), timeout=2)
alice, bob, eve, chat = c.bootstrap()
attachment, content = c.attachment(alice, chat)
c.wait_ready(alice, attachment)
expected = [c.message(alice, chat, "Before failures")["json"]]
targets = containers()
assert len(targets) == 2
results = []
try:
for container in targets:
target = instance_id(container)
env = {**os.environ, "TARGET_INSTANCE_ID": target}
started = time.monotonic()
process = subprocess.Popen(
[str(ROOT / "scripts/stop-one.sh")],
cwd=ROOT,
env=env,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
first = None
failures = 0
served = []
success = 0
key = uuid.uuid4().hex
text = "During failure " + key
while time.monotonic() - started < 15:
assert process.poll() in (None, 0)
try:
response = c.message(alice, chat, text, key=key)
expected.append(response["json"])
success += 1
if process.poll() == 0:
if first is None:
first = time.monotonic() - started
served.append(response["headers"]["X-Instance-Id"])
key = uuid.uuid4().hex
text = "During failure " + key
except Failure:
failures += 1
time.sleep(0.15)
assert process.wait(timeout=1) == 0
assert first is not None and first <= 10
assert len(served) >= 5 and target not in served[-5:]
assert not describe(container)["Running"]
messages = history(c, bob, chat)
assert len({x["id"] for x in messages}) == len(messages)
assert len({x["text"] for x in messages}) == len(messages)
assert all(x in messages for x in expected)
assert (
c.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob)[
"bytes"
]
== content
)
results.append(
{
"container": container[:12],
"target_instance": target,
"successful_writes": success,
"failed_attempts": failures,
"first_recovery_seconds": round(first, 3),
"result": "PASS",
}
)
compose("start", "api")
wait_ready(container)
time.sleep(2)
finally:
compose("start", "api")
(ROOT / "evidence/failover-summary.json").write_text(
json.dumps(
{
"result": "PASS",
"distinct_containers": len(results),
"runs": len(results),
"events": results,
},
indent=2,
)
+ "\n"
)
print(
"PASS: both exact API containers stopped under writes; sessions, messages and attachment preserved"
)
if __name__ == "__main__":
main()