Files
mtusi2026lab3/student-tests/rolling_update_experiment.py
T

92 lines
3.5 KiB
Python

"""Run concurrent writes while both API replicas are replaced one at a time."""
import concurrent.futures
import json
import math
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
def main():
args = arguments("Rolling update under write load")
base = Client(args.base_url, args.ca_file)
alice, _, _, chat = base.bootstrap()
path = "/api/v1/chats/" + chat["id"] + "/messages"
workers = 4
start_barrier = threading.Barrier(workers + 1)
stop = threading.Event()
def worker(worker_id):
client = Client(args.base_url, args.ca_file, timeout=3)
successes = failures = 0
latencies = []
instances = set()
index = 0
start_barrier.wait()
while not stop.is_set():
key = uuid.uuid4().hex
started = time.monotonic()
try:
result = client.call("POST", path, 201, actor=alice,
body={"text": "rolling-%d-%d" % (worker_id, index)},
headers={"Idempotency-Key": key}, schema="Message")
successes += 1
latencies.append(time.monotonic() - started)
instances.add(result["headers"]["X-Instance-Id"])
except Failure:
failures += 1
index += 1
return successes, failures, latencies, instances
started = time.monotonic()
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
futures = [pool.submit(worker, i) for i in range(workers)]
start_barrier.wait()
update = subprocess.run([str(ROOT / "scripts" / "rolling-update.sh")],
cwd=ROOT, stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE, text=True, timeout=120)
time.sleep(3)
stop.set()
results = [future.result() for future in futures]
elapsed = time.monotonic() - started
check(update.returncode == 0, "rolling update failed: " + update.stderr[-500:])
successes = sum(item[0] for item in results)
failures = sum(item[1] for item in results)
latencies = sorted(value for item in results for value in item[2])
instances = sorted(set().union(*(item[3] for item in results)))
check(successes > 0, "no successful writes during rolling update")
check(set(instances) == {"api-1", "api-2"},
"both replicas were not observed during rolling update")
def percentile(fraction):
if not latencies:
return None
pos = max(0, math.ceil(fraction * len(latencies)) - 1)
return round(latencies[pos] * 1000, 3)
output = {
"workload": "four-client closed-loop POST messages",
"elapsed_seconds": round(elapsed, 3),
"successful_requests": successes,
"failed_requests": failures,
"successful_rps": round(successes / elapsed, 3),
"latency_ms_successes": {"p50": percentile(.5), "p95": percentile(.95),
"p99": percentile(.99)},
"instances": instances,
}
(ROOT / "evidence").mkdir(exist_ok=True)
(ROOT / "evidence" / "rolling-update.json").write_text(
json.dumps(output, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print("PASS:", json.dumps(output, ensure_ascii=False))
if __name__ == "__main__":
run(main)