ЛР5: устойчивый versioned pipeline изображений, outbox и доказательства отказов
This commit is contained in:
@@ -0,0 +1,71 @@
|
||||
"""Record real restart scope and exercise failover of both physical API containers."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def run(args):
|
||||
result = subprocess.run(args, text=True, capture_output=True, timeout=180)
|
||||
if result.returncode:
|
||||
raise RuntimeError("Verification failed:\n" + result.stdout + result.stderr[-2000:])
|
||||
return result
|
||||
|
||||
|
||||
def snapshot():
|
||||
ids = run(["docker", "compose", "ps", "-aq"]).stdout.split()
|
||||
items = json.loads(run(["docker", "inspect", *ids]).stdout)
|
||||
return [
|
||||
{"id": item["Id"], "service": item["Config"]["Labels"]["com.docker.compose.service"],
|
||||
"started_at": item["State"]["StartedAt"], "status": item["State"]["Status"],
|
||||
"mounts": [{"type": mount["Type"], "name": mount.get("Name"), "destination": mount["Destination"]}
|
||||
for mount in item["Mounts"] if mount["Type"] == "volume"]}
|
||||
for item in items
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
base = os.getenv("BASE_URL", "http://127.0.0.1:8085")
|
||||
before = snapshot()
|
||||
result = run(["python3", "tests/resilience.py", "--mode", "persistence", "--action-script", "scripts/restart.sh", "--base-url", base])
|
||||
Path("evidence/persistence-final.txt").write_text(result.stdout)
|
||||
after = snapshot()
|
||||
changed = []
|
||||
by_id = {item["id"]: item for item in before}
|
||||
for item in after:
|
||||
old = by_id.get(item["id"])
|
||||
if old and old["started_at"] != item["started_at"]:
|
||||
changed.append(item["service"])
|
||||
required = {"postgres", "cache", "broker", "storage", "s3", "api", "lb", "worker", "dispatcher"}
|
||||
assert required <= set(changed), (required, changed)
|
||||
assert {json.dumps(m, sort_keys=True) for i in before for m in i["mounts"]} == {json.dumps(m, sort_keys=True) for i in after for m in i["mounts"]}
|
||||
Path("evidence/persistence-runtime.json").write_text(json.dumps({"before": before, "after": after, "restarted_services": changed, "volumes_unchanged": True, "passed": True}, indent=2))
|
||||
|
||||
actual_api = {item["id"] for item in after if item["service"] == "api"}
|
||||
assert len(actual_api) >= 2
|
||||
stopped = set()
|
||||
attempts = []
|
||||
try:
|
||||
for _ in range(6):
|
||||
result = run(["python3", "tests/resilience.py", "--mode", "failover", "--action-script", "scripts/stop-one.sh", "--base-url", base])
|
||||
receipt = json.loads(Path("evidence/last-stopped.json").read_text())
|
||||
assert receipt["stopped_container"] in actual_api
|
||||
state = json.loads(run(["docker", "inspect", receipt["stopped_container"]]).stdout)[0]
|
||||
assert not state["State"]["Running"]
|
||||
summaries = [json.loads(line) for line in result.stdout.splitlines() if line.startswith("{")]
|
||||
attempts.append({"stopped": receipt, "summary": summaries, "stdout": result.stdout})
|
||||
stopped.add(receipt["stopped_container"])
|
||||
run(["docker", "compose", "start", "api"])
|
||||
run(["docker", "compose", "up", "-d", "--no-recreate", "--wait", "api", "lb"])
|
||||
if stopped == actual_api:
|
||||
break
|
||||
assert stopped == actual_api, "Both physical replicas were not covered"
|
||||
finally:
|
||||
run(["docker", "compose", "start", "api"])
|
||||
Path("evidence/failover-final.json").write_text(json.dumps({"actual_api_container_ids": sorted(actual_api), "covered_container_ids": sorted(stopped), "attempts": attempts, "passed": True}, indent=2))
|
||||
print("PASS: full restart and failover of both physical API replicas")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user