Files
mtusi2026lab5/tests/pipeline_faults.py
T
2026-11-19 19:45:26 +03:00

62 lines
2.8 KiB
Python

#!/usr/bin/env python3
"""Exercise a worker crash window and verify idempotent recovery."""
import json
from pathlib import Path
import subprocess
import time
from client import Client, arguments, check, run
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file, timeout=10)
alice, _bob, _eve, chat = client.bootstrap()
root = Path(__file__).resolve().parents[1]
evidence = root / "evidence" / "pipeline-faults.json"
subprocess.run(["docker", "compose", "stop", "worker"], cwd=root, check=True,
stdout=subprocess.DEVNULL)
try:
attachment, _ = client.attachment(alice, chat)
fault_name = "lab5-fault-worker"
subprocess.run(["docker", "rm", "-f", fault_name], cwd=root,
check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
subprocess.run(
["docker", "compose", "run", "-d", "--no-deps", "--name", fault_name,
"-e", "FAULT_INJECTION=after-side-effect", "-e", "JOB_TIMEOUT_SECONDS=5",
"worker"],
cwd=root, check=True, stdout=subprocess.DEVNULL,
)
time.sleep(7)
subprocess.run(["docker", "rm", "-f", fault_name], cwd=root, check=False,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
recovery_name = "lab5-recovery-worker"
subprocess.run(["docker", "rm", "-f", recovery_name], cwd=root, check=False,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
subprocess.run(
["docker", "compose", "run", "-d", "--no-deps", "--name", recovery_name,
"-e", "JOB_TIMEOUT_SECONDS=5", "worker"],
cwd=root, check=True, stdout=subprocess.DEVNULL,
)
ready = client.wait_ready(alice, attachment, wait_seconds=60)
variants = {item["name"]: item for item in ready["variants"]}
check(len(variants) == 3, "recovery did not publish exactly three variants")
result = {
"fault": "after-side-effect",
"attachment_id": attachment["id"],
"status": ready["status"],
"variant_names": sorted(variants),
"invariant": "deterministic effect keys produced one logical result",
}
evidence.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
print("PASS: fault injection recovered idempotently")
finally:
subprocess.run(["docker", "rm", "-f", "lab5-recovery-worker"], cwd=root,
check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
subprocess.run(["docker", "compose", "start", "worker"], cwd=root, check=False,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
if __name__ == "__main__":
run(main)