forked from mtusi2026/mtusi2026lab5
62 lines
2.8 KiB
Python
62 lines
2.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Exercise a worker crash window and verify idempotent recovery."""
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
|
|
from client import Client, arguments, check, run
|
|
|
|
|
|
def main():
|
|
args = arguments(__doc__)
|
|
client = Client(args.base_url, args.ca_file, timeout=10)
|
|
alice, _bob, _eve, chat = client.bootstrap()
|
|
root = Path(__file__).resolve().parents[1]
|
|
evidence = root / "evidence" / "pipeline-faults.json"
|
|
subprocess.run(["docker", "compose", "stop", "worker"], cwd=root, check=True,
|
|
stdout=subprocess.DEVNULL)
|
|
try:
|
|
attachment, _ = client.attachment(alice, chat)
|
|
fault_name = "lab5-fault-worker"
|
|
subprocess.run(["docker", "rm", "-f", fault_name], cwd=root,
|
|
check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
subprocess.run(
|
|
["docker", "compose", "run", "-d", "--no-deps", "--name", fault_name,
|
|
"-e", "FAULT_INJECTION=after-side-effect", "-e", "JOB_TIMEOUT_SECONDS=5",
|
|
"worker"],
|
|
cwd=root, check=True, stdout=subprocess.DEVNULL,
|
|
)
|
|
time.sleep(7)
|
|
subprocess.run(["docker", "rm", "-f", fault_name], cwd=root, check=False,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
recovery_name = "lab5-recovery-worker"
|
|
subprocess.run(["docker", "rm", "-f", recovery_name], cwd=root, check=False,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
subprocess.run(
|
|
["docker", "compose", "run", "-d", "--no-deps", "--name", recovery_name,
|
|
"-e", "JOB_TIMEOUT_SECONDS=5", "worker"],
|
|
cwd=root, check=True, stdout=subprocess.DEVNULL,
|
|
)
|
|
ready = client.wait_ready(alice, attachment, wait_seconds=60)
|
|
variants = {item["name"]: item for item in ready["variants"]}
|
|
check(len(variants) == 3, "recovery did not publish exactly three variants")
|
|
result = {
|
|
"fault": "after-side-effect",
|
|
"attachment_id": attachment["id"],
|
|
"status": ready["status"],
|
|
"variant_names": sorted(variants),
|
|
"invariant": "deterministic effect keys produced one logical result",
|
|
}
|
|
evidence.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
|
|
print("PASS: fault injection recovered idempotently")
|
|
finally:
|
|
subprocess.run(["docker", "rm", "-f", "lab5-recovery-worker"], cwd=root,
|
|
check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
subprocess.run(["docker", "compose", "start", "worker"], cwd=root, check=False,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|