forked from mtusi2026/mtusi2026lab3
92 lines
3.5 KiB
Python
92 lines
3.5 KiB
Python
"""Run concurrent writes while both API replicas are replaced one at a time."""
|
|
import concurrent.futures
|
|
import json
|
|
import math
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure, arguments, check, run # noqa: E402
|
|
|
|
|
|
def main():
|
|
args = arguments("Rolling update under write load")
|
|
base = Client(args.base_url, args.ca_file)
|
|
alice, _, _, chat = base.bootstrap()
|
|
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
|
workers = 4
|
|
start_barrier = threading.Barrier(workers + 1)
|
|
stop = threading.Event()
|
|
|
|
def worker(worker_id):
|
|
client = Client(args.base_url, args.ca_file, timeout=3)
|
|
successes = failures = 0
|
|
latencies = []
|
|
instances = set()
|
|
index = 0
|
|
start_barrier.wait()
|
|
while not stop.is_set():
|
|
key = uuid.uuid4().hex
|
|
started = time.monotonic()
|
|
try:
|
|
result = client.call("POST", path, 201, actor=alice,
|
|
body={"text": "rolling-%d-%d" % (worker_id, index)},
|
|
headers={"Idempotency-Key": key}, schema="Message")
|
|
successes += 1
|
|
latencies.append(time.monotonic() - started)
|
|
instances.add(result["headers"]["X-Instance-Id"])
|
|
except Failure:
|
|
failures += 1
|
|
index += 1
|
|
return successes, failures, latencies, instances
|
|
|
|
started = time.monotonic()
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
|
|
futures = [pool.submit(worker, i) for i in range(workers)]
|
|
start_barrier.wait()
|
|
update = subprocess.run([str(ROOT / "scripts" / "rolling-update.sh")],
|
|
cwd=ROOT, stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.PIPE, text=True, timeout=120)
|
|
time.sleep(3)
|
|
stop.set()
|
|
results = [future.result() for future in futures]
|
|
elapsed = time.monotonic() - started
|
|
check(update.returncode == 0, "rolling update failed: " + update.stderr[-500:])
|
|
successes = sum(item[0] for item in results)
|
|
failures = sum(item[1] for item in results)
|
|
latencies = sorted(value for item in results for value in item[2])
|
|
instances = sorted(set().union(*(item[3] for item in results)))
|
|
check(successes > 0, "no successful writes during rolling update")
|
|
check(set(instances) == {"api-1", "api-2"},
|
|
"both replicas were not observed during rolling update")
|
|
|
|
def percentile(fraction):
|
|
if not latencies:
|
|
return None
|
|
pos = max(0, math.ceil(fraction * len(latencies)) - 1)
|
|
return round(latencies[pos] * 1000, 3)
|
|
|
|
output = {
|
|
"workload": "four-client closed-loop POST messages",
|
|
"elapsed_seconds": round(elapsed, 3),
|
|
"successful_requests": successes,
|
|
"failed_requests": failures,
|
|
"successful_rps": round(successes / elapsed, 3),
|
|
"latency_ms_successes": {"p50": percentile(.5), "p95": percentile(.95),
|
|
"p99": percentile(.99)},
|
|
"instances": instances,
|
|
}
|
|
(ROOT / "evidence").mkdir(exist_ok=True)
|
|
(ROOT / "evidence" / "rolling-update.json").write_text(
|
|
json.dumps(output, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
print("PASS:", json.dumps(output, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|