72 lines
3.5 KiB
Python
72 lines
3.5 KiB
Python
"""Record real restart scope and exercise failover of both physical API containers."""
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
|
|
def run(args):
|
|
result = subprocess.run(args, text=True, capture_output=True, timeout=180)
|
|
if result.returncode:
|
|
raise RuntimeError("Verification failed:\n" + result.stdout + result.stderr[-2000:])
|
|
return result
|
|
|
|
|
|
def snapshot():
|
|
ids = run(["docker", "compose", "ps", "-aq"]).stdout.split()
|
|
items = json.loads(run(["docker", "inspect", *ids]).stdout)
|
|
return [
|
|
{"id": item["Id"], "service": item["Config"]["Labels"]["com.docker.compose.service"],
|
|
"started_at": item["State"]["StartedAt"], "status": item["State"]["Status"],
|
|
"mounts": [{"type": mount["Type"], "name": mount.get("Name"), "destination": mount["Destination"]}
|
|
for mount in item["Mounts"] if mount["Type"] == "volume"]}
|
|
for item in items
|
|
]
|
|
|
|
|
|
def main():
|
|
base = os.getenv("BASE_URL", "http://127.0.0.1:8085")
|
|
before = snapshot()
|
|
result = run(["python3", "tests/resilience.py", "--mode", "persistence", "--action-script", "scripts/restart.sh", "--base-url", base])
|
|
Path("evidence/persistence-final.txt").write_text(result.stdout)
|
|
after = snapshot()
|
|
changed = []
|
|
by_id = {item["id"]: item for item in before}
|
|
for item in after:
|
|
old = by_id.get(item["id"])
|
|
if old and old["started_at"] != item["started_at"]:
|
|
changed.append(item["service"])
|
|
required = {"postgres", "cache", "broker", "storage", "s3", "api", "lb", "worker", "dispatcher"}
|
|
assert required <= set(changed), (required, changed)
|
|
assert {json.dumps(m, sort_keys=True) for i in before for m in i["mounts"]} == {json.dumps(m, sort_keys=True) for i in after for m in i["mounts"]}
|
|
Path("evidence/persistence-runtime.json").write_text(json.dumps({"before": before, "after": after, "restarted_services": changed, "volumes_unchanged": True, "passed": True}, indent=2))
|
|
|
|
actual_api = {item["id"] for item in after if item["service"] == "api"}
|
|
assert len(actual_api) >= 2
|
|
stopped = set()
|
|
attempts = []
|
|
try:
|
|
for _ in range(6):
|
|
result = run(["python3", "tests/resilience.py", "--mode", "failover", "--action-script", "scripts/stop-one.sh", "--base-url", base])
|
|
receipt = json.loads(Path("evidence/last-stopped.json").read_text())
|
|
assert receipt["stopped_container"] in actual_api
|
|
state = json.loads(run(["docker", "inspect", receipt["stopped_container"]]).stdout)[0]
|
|
assert not state["State"]["Running"]
|
|
summaries = [json.loads(line) for line in result.stdout.splitlines() if line.startswith("{")]
|
|
attempts.append({"stopped": receipt, "summary": summaries, "stdout": result.stdout})
|
|
stopped.add(receipt["stopped_container"])
|
|
run(["docker", "compose", "start", "api"])
|
|
run(["docker", "compose", "up", "-d", "--no-recreate", "--wait", "api", "lb"])
|
|
if stopped == actual_api:
|
|
break
|
|
assert stopped == actual_api, "Both physical replicas were not covered"
|
|
finally:
|
|
run(["docker", "compose", "start", "api"])
|
|
Path("evidence/failover-final.json").write_text(json.dumps({"actual_api_container_ids": sorted(actual_api), "covered_container_ids": sorted(stopped), "attempts": attempts, "passed": True}, indent=2))
|
|
print("PASS: full restart and failover of both physical API replicas")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|